From 4fb7d12df63a062a865daa1008cbed879524eb2b Mon Sep 17 00:00:00 2001 From: Mohamed Mady Date: Tue, 23 Jun 2026 23:55:42 +0200 Subject: [PATCH] =?UTF-8?q?Remove=20DeBERTa-V22-Seed42=20remnants=20?= =?UTF-8?q?=E2=80=94=20metadata=20missing=20causes=20bot=20failure=20(fixe?= =?UTF-8?q?s=20#167=20#169)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DeBERTa-V22-Seed42/results.json | 217721 --------------- 1 file changed, 217721 deletions(-) delete mode 100644 leaderboard/submissions/DeBERTa-V22-Seed42/results.json diff --git a/leaderboard/submissions/DeBERTa-V22-Seed42/results.json b/leaderboard/submissions/DeBERTa-V22-Seed42/results.json deleted file mode 100644 index ec05f8da..00000000 --- a/leaderboard/submissions/DeBERTa-V22-Seed42/results.json +++ /dev/null @@ -1,217721 +0,0 @@ -{ - "_autogen_note": "This file is automatically generated by the RAID leaderboard submission script. Do not edit this file manually or include it in new submissions' PRs.", - "_submission_hash": "82b358dccab5fcd85cfa18ff2d3435edaf7161d78d2e22576bf9496be6a11ce8", - "_results_hash": "7b20161b8db8b0bf8ff4710cc92a406f0e75a9e131ba7009dbbd0c3295dc4fc9", - "date_released": "2026-05-01", - "detector_name": "DeBERTa-ConPara-v2.2-Seed42", - "contact_info": "Email Address: mohamed.mady@oth-regensburg.de", - "website": "", - "paper_link": "", - "huggingface_link": "", - "github_link": "", - "additional_metadata": null, - "score_agg": { - "all": { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 635121, - "fn": 17679, - "accuracy": 0.9729181985294117 - }, - "0.01": { - "tp": 610674, - "fn": 42126, - "accuracy": 0.93546875 - } - }, - "auroc": 0.9686252995509728 - }, - "no_adversarial": { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 52781, - "fn": 1619, - "accuracy": 0.9702389705882353 - }, - "0.01": { - "tp": 50802, - "fn": 3598, - "accuracy": 0.933860294117647 - } - }, - "auroc": 0.9704708505667892 - } - }, - "scores": [ - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998906249999999 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999453125 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998468749999999 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9981510416666668 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9989989583333334 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999234375 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9990208333333335 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9994721354166667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9823104166666665 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9834187500000001 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9828645833333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9140593750000001 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9689864583333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9415229166666667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9481848958333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9762026041666667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 777, - "fn": 23, - "accuracy": 0.97125 - }, - "0.01": { - "tp": 766, - "fn": 34, - "accuracy": 0.9575 - } - }, - "auroc": 0.9621937500000001 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9976750000000001 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.996028125 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9968515625 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985020833333333 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9878739583333334 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9931880208333332 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9980885416666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9919510416666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9950197916666668 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9753906250000001 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998325 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9868578125 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.963790625 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9453947916666666 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9545927083333334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9695906250000002 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9718598958333332 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 783, - "fn": 17, - "accuracy": 0.97875 - } - }, - "auroc": 0.9707252604166667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9959072916666667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.974028125 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9849677083333334 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9127208333333332 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.924065625 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.9183932291666668 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9543140625 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9490468750000001 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 769, - "fn": 31, - "accuracy": 0.96125 - }, - "0.01": { - "tp": 752, - "fn": 48, - "accuracy": 0.94 - } - }, - "auroc": 0.95168046875 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994447916666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9969697916666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9982072916666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9971687499999999 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.971071875 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9841203125000001 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983067708333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9840208333333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - } - }, - "auroc": 0.9911638020833333 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9331260416666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9331260416666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9207979166666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9207979166666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9269619791666668 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9269619791666668 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - }, - "0.01": { - "tp": 137, - "fn": 63, - "accuracy": 0.685 - } - }, - "auroc": 0.863309375 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - }, - "0.01": { - "tp": 137, - "fn": 63, - "accuracy": 0.685 - } - }, - "auroc": 0.863309375 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 152, - "fn": 48, - "accuracy": 0.76 - }, - "0.01": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - } - }, - "auroc": 0.8356281249999999 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 152, - "fn": 48, - "accuracy": 0.76 - }, - "0.01": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - } - }, - "auroc": 0.8356281249999999 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 316, - "fn": 84, - "accuracy": 0.79 - }, - "0.01": { - "tp": 271, - "fn": 129, - "accuracy": 0.6775 - } - }, - "auroc": 0.84946875 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 316, - "fn": 84, - "accuracy": 0.79 - }, - "0.01": { - "tp": 271, - "fn": 129, - "accuracy": 0.6775 - } - }, - "auroc": 0.84946875 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9991708333333333 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9991708333333333 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995765624999999 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995765624999999 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9773729166666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9773729166666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9886854166666668 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9886854166666668 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9651770833333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9651770833333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.945515625 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.945515625 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9553463541666667 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9553463541666667 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2144, - "fn": 56, - "accuracy": 0.9745454545454545 - }, - "0.01": { - "tp": 2103, - "fn": 97, - "accuracy": 0.9559090909090909 - } - }, - "auroc": 0.9738473484848484 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1194, - "fn": 6, - "accuracy": 0.995 - } - }, - "auroc": 0.9914434027777779 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3341, - "fn": 59, - "accuracy": 0.9826470588235294 - }, - "0.01": { - "tp": 3297, - "fn": 103, - "accuracy": 0.9697058823529412 - } - }, - "auroc": 0.9800577205882353 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2092, - "fn": 108, - "accuracy": 0.9509090909090909 - }, - "0.01": { - "tp": 2035, - "fn": 165, - "accuracy": 0.925 - } - }, - "auroc": 0.9513249053030302 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1176, - "fn": 24, - "accuracy": 0.98 - }, - "0.01": { - "tp": 1154, - "fn": 46, - "accuracy": 0.9616666666666667 - } - }, - "auroc": 0.9659239583333334 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3268, - "fn": 132, - "accuracy": 0.9611764705882353 - }, - "0.01": { - "tp": 3189, - "fn": 211, - "accuracy": 0.9379411764705883 - } - }, - "auroc": 0.9564775122549021 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 4236, - "fn": 164, - "accuracy": 0.9627272727272728 - }, - "0.01": { - "tp": 4138, - "fn": 262, - "accuracy": 0.9404545454545454 - } - }, - "auroc": 0.9625861268939393 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2373, - "fn": 27, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 2348, - "fn": 52, - "accuracy": 0.9783333333333334 - } - }, - "auroc": 0.9786836805555555 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 6609, - "fn": 191, - "accuracy": 0.9719117647058824 - }, - "0.01": { - "tp": 6486, - "fn": 314, - "accuracy": 0.9538235294117647 - } - }, - "auroc": 0.9682676164215686 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998906249999999 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999453125 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998468749999999 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9981510416666668 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9989989583333334 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999234375 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9990208333333335 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9994721354166667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9823104166666665 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9834187500000001 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9828645833333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9140593750000001 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9689864583333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9415229166666667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9481848958333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9762026041666667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 777, - "fn": 23, - "accuracy": 0.97125 - }, - "0.01": { - "tp": 766, - "fn": 34, - "accuracy": 0.9575 - } - }, - "auroc": 0.9621937500000001 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9976750000000001 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.996028125 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9968515625 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985020833333333 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9878739583333334 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9931880208333332 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9980885416666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9919510416666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9950197916666668 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9753906250000001 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998325 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9868578125 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.963790625 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9453947916666666 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9545927083333334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9695906250000002 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9718598958333332 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 783, - "fn": 17, - "accuracy": 0.97875 - } - }, - "auroc": 0.9707252604166667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9959072916666667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.974028125 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9849677083333334 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9127208333333332 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.924065625 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.9183932291666668 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9543140625 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9490468750000001 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 769, - "fn": 31, - "accuracy": 0.96125 - }, - "0.01": { - "tp": 752, - "fn": 48, - "accuracy": 0.94 - } - }, - "auroc": 0.95168046875 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994447916666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9969697916666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9982072916666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9971687499999999 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.971071875 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9841203125000001 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983067708333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9840208333333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - } - }, - "auroc": 0.9911638020833333 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9331260416666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9331260416666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9207979166666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9207979166666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9269619791666668 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9269619791666668 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - }, - "0.01": { - "tp": 137, - "fn": 63, - "accuracy": 0.685 - } - }, - "auroc": 0.863309375 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - }, - "0.01": { - "tp": 137, - "fn": 63, - "accuracy": 0.685 - } - }, - "auroc": 0.863309375 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 152, - "fn": 48, - "accuracy": 0.76 - }, - "0.01": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - } - }, - "auroc": 0.8356281249999999 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 152, - "fn": 48, - "accuracy": 0.76 - }, - "0.01": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - } - }, - "auroc": 0.8356281249999999 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 316, - "fn": 84, - "accuracy": 0.79 - }, - "0.01": { - "tp": 271, - "fn": 129, - "accuracy": 0.6775 - } - }, - "auroc": 0.84946875 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 316, - "fn": 84, - "accuracy": 0.79 - }, - "0.01": { - "tp": 271, - "fn": 129, - "accuracy": 0.6775 - } - }, - "auroc": 0.84946875 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9991708333333333 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9991708333333333 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995765624999999 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995765624999999 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9773729166666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9773729166666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9886854166666668 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9886854166666668 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9651770833333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9651770833333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.945515625 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.945515625 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9553463541666667 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9553463541666667 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2144, - "fn": 56, - "accuracy": 0.9745454545454545 - }, - "0.01": { - "tp": 2103, - "fn": 97, - "accuracy": 0.9559090909090909 - } - }, - "auroc": 0.9738473484848484 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1194, - "fn": 6, - "accuracy": 0.995 - } - }, - "auroc": 0.9914434027777779 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3341, - "fn": 59, - "accuracy": 0.9826470588235294 - }, - "0.01": { - "tp": 3297, - "fn": 103, - "accuracy": 0.9697058823529412 - } - }, - "auroc": 0.9800577205882353 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2092, - "fn": 108, - "accuracy": 0.9509090909090909 - }, - "0.01": { - "tp": 2035, - "fn": 165, - "accuracy": 0.925 - } - }, - "auroc": 0.9513249053030302 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1176, - "fn": 24, - "accuracy": 0.98 - }, - "0.01": { - "tp": 1154, - "fn": 46, - "accuracy": 0.9616666666666667 - } - }, - "auroc": 0.9659239583333334 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3268, - "fn": 132, - "accuracy": 0.9611764705882353 - }, - "0.01": { - "tp": 3189, - "fn": 211, - "accuracy": 0.9379411764705883 - } - }, - "auroc": 0.9564775122549021 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 4236, - "fn": 164, - "accuracy": 0.9627272727272728 - }, - "0.01": { - "tp": 4138, - "fn": 262, - "accuracy": 0.9404545454545454 - } - }, - "auroc": 0.9625861268939393 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2373, - "fn": 27, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 2348, - "fn": 52, - "accuracy": 0.9783333333333334 - } - }, - "auroc": 0.9786836805555555 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 6609, - "fn": 191, - "accuracy": 0.9719117647058824 - }, - "0.01": { - "tp": 6486, - "fn": 314, - "accuracy": 0.9538235294117647 - } - }, - "auroc": 0.9682676164215686 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999218750000001 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998729166666667 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998973958333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997708333333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9988208333333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992958333333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998463541666667 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999346875 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995966145833333 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9979270833333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.982884375 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9904057291666668 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9635906249999999 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9693625 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9664765624999999 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9807588541666666 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9761234375 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - }, - "0.01": { - "tp": 789, - "fn": 11, - "accuracy": 0.98625 - } - }, - "auroc": 0.9784411458333333 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9975989583333333 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9968020833333334 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9972005208333333 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998890625 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9895708333333334 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9942307291666668 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9982447916666666 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9931864583333334 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.995715625 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9915614583333332 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992114583333334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9953864583333334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9883802083333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9839270833333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9861536458333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9899708333333332 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9915692708333335 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - } - }, - "auroc": 0.9907700520833334 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9972333333333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9946489583333332 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9959411458333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9673135416666667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9583375000000001 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9628255208333334 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9822734375 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9764932291666667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 784, - "fn": 16, - "accuracy": 0.98 - } - }, - "auroc": 0.9793833333333334 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996989583333332 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9977708333333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987348958333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981895833333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9838729166666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.99103125 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989442708333334 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.990821875 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9948830729166667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9553718749999999 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9553718749999999 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9532375000000001 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9532375000000001 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9543046875000001 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9543046875000001 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9432875000000001 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9432875000000001 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9306406249999999 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9306406249999999 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9369640625000001 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9369640625000001 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999895833333332 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999895833333332 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987895833333333 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987895833333333 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993895833333334 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993895833333334 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999471875 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999471875 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9934791666666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9934791666666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9964755208333334 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9964755208333334 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9835739583333334 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9835739583333334 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9732854166666667 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9732854166666667 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9784296875 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9784296875 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2179, - "fn": 21, - "accuracy": 0.9904545454545455 - }, - "0.01": { - "tp": 2172, - "fn": 28, - "accuracy": 0.9872727272727273 - } - }, - "auroc": 0.9877851325757575 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - } - }, - "auroc": 0.9951984375 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3376, - "fn": 24, - "accuracy": 0.9929411764705882 - }, - "0.01": { - "tp": 3369, - "fn": 31, - "accuracy": 0.9908823529411764 - } - }, - "auroc": 0.990401593137255 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2181, - "fn": 19, - "accuracy": 0.9913636363636363 - }, - "0.01": { - "tp": 2161, - "fn": 39, - "accuracy": 0.9822727272727273 - } - }, - "auroc": 0.9786879734848485 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1191, - "fn": 9, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 1178, - "fn": 22, - "accuracy": 0.9816666666666667 - } - }, - "auroc": 0.9806486111111111 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3372, - "fn": 28, - "accuracy": 0.991764705882353 - }, - "0.01": { - "tp": 3339, - "fn": 61, - "accuracy": 0.9820588235294118 - } - }, - "auroc": 0.9793799632352942 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 4360, - "fn": 40, - "accuracy": 0.990909090909091 - }, - "0.01": { - "tp": 4333, - "fn": 67, - "accuracy": 0.9847727272727272 - } - }, - "auroc": 0.9832365530303031 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2388, - "fn": 12, - "accuracy": 0.995 - }, - "0.01": { - "tp": 2375, - "fn": 25, - "accuracy": 0.9895833333333334 - } - }, - "auroc": 0.9879235243055555 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 6748, - "fn": 52, - "accuracy": 0.9923529411764705 - }, - "0.01": { - "tp": 6708, - "fn": 92, - "accuracy": 0.9864705882352941 - } - }, - "auroc": 0.9848907781862747 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999871875 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99766875 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987703125 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998364583333332 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9947677083333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9973020833333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998541666666667 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9962182291666667 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980361979166666 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9801375 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9815562500000001 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9808468750000001 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.90521875 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9643947916666668 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - } - }, - "auroc": 0.9348067708333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.942678125 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9729755208333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 775, - "fn": 25, - "accuracy": 0.96875 - }, - "0.01": { - "tp": 760, - "fn": 40, - "accuracy": 0.95 - } - }, - "auroc": 0.9578268229166667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9879625 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9854125 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9866874999999999 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9902395833333333 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.973225 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9817322916666666 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9891010416666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.97931875 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - } - }, - "auroc": 0.9842098958333334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.974240625 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9920208333333334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9831307291666667 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.95336875 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9458239583333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9495963541666667 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9638046875000001 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9689223958333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 782, - "fn": 18, - "accuracy": 0.9775 - } - }, - "auroc": 0.9663635416666666 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9950145833333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.96398125 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9794979166666667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.8995479166666667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9256166666666666 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.9125822916666666 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9472812500000001 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9447989583333334 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 752, - "fn": 48, - "accuracy": 0.94 - } - }, - "auroc": 0.9460401041666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9945416666666668 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9891604166666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9918510416666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99068125 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9556510416666668 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9731661458333334 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9926114583333334 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9724057291666668 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 785, - "fn": 15, - "accuracy": 0.98125 - } - }, - "auroc": 0.98250859375 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - }, - "0.01": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - } - }, - "auroc": 0.9013583333333334 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - }, - "0.01": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - } - }, - "auroc": 0.9013583333333334 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.8814124999999999 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.8814124999999999 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 324, - "fn": 76, - "accuracy": 0.81 - } - }, - "auroc": 0.8913854166666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 324, - "fn": 76, - "accuracy": 0.81 - } - }, - "auroc": 0.8913854166666667 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 142, - "fn": 58, - "accuracy": 0.71 - }, - "0.01": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - } - }, - "auroc": 0.8351354166666667 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 142, - "fn": 58, - "accuracy": 0.71 - }, - "0.01": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - } - }, - "auroc": 0.8351354166666667 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 140, - "fn": 60, - "accuracy": 0.7 - }, - "0.01": { - "tp": 114, - "fn": 86, - "accuracy": 0.57 - } - }, - "auroc": 0.7934322916666666 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 140, - "fn": 60, - "accuracy": 0.7 - }, - "0.01": { - "tp": 114, - "fn": 86, - "accuracy": 0.57 - } - }, - "auroc": 0.7934322916666666 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 282, - "fn": 118, - "accuracy": 0.705 - }, - "0.01": { - "tp": 233, - "fn": 167, - "accuracy": 0.5825 - } - }, - "auroc": 0.8142838541666667 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 282, - "fn": 118, - "accuracy": 0.705 - }, - "0.01": { - "tp": 233, - "fn": 167, - "accuracy": 0.5825 - } - }, - "auroc": 0.8142838541666667 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9938375 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9938375 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9890125 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9890125 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.991425 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.991425 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9967072916666666 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9967072916666666 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9470770833333333 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9470770833333333 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9718921875 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9718921875 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9498385416666667 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9498385416666667 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - } - }, - "auroc": 0.9279864583333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - } - }, - "auroc": 0.9279864583333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9389125 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9389125 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2110, - "fn": 90, - "accuracy": 0.9590909090909091 - }, - "0.01": { - "tp": 2058, - "fn": 142, - "accuracy": 0.9354545454545454 - } - }, - "auroc": 0.9644223484848484 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1195, - "fn": 5, - "accuracy": 0.9958333333333333 - }, - "0.01": { - "tp": 1192, - "fn": 8, - "accuracy": 0.9933333333333333 - } - }, - "auroc": 0.9849666666666667 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3305, - "fn": 95, - "accuracy": 0.9720588235294118 - }, - "0.01": { - "tp": 3250, - "fn": 150, - "accuracy": 0.9558823529411765 - } - }, - "auroc": 0.9716732843137255 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2052, - "fn": 148, - "accuracy": 0.9327272727272727 - }, - "0.01": { - "tp": 1971, - "fn": 229, - "accuracy": 0.8959090909090909 - } - }, - "auroc": 0.9343466856060607 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1176, - "fn": 24, - "accuracy": 0.98 - }, - "0.01": { - "tp": 1146, - "fn": 54, - "accuracy": 0.955 - } - }, - "auroc": 0.9599131944444446 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3228, - "fn": 172, - "accuracy": 0.9494117647058824 - }, - "0.01": { - "tp": 3117, - "fn": 283, - "accuracy": 0.9167647058823529 - } - }, - "auroc": 0.9433701593137254 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 4162, - "fn": 238, - "accuracy": 0.9459090909090909 - }, - "0.01": { - "tp": 4029, - "fn": 371, - "accuracy": 0.9156818181818182 - } - }, - "auroc": 0.9493845170454546 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2371, - "fn": 29, - "accuracy": 0.9879166666666667 - }, - "0.01": { - "tp": 2338, - "fn": 62, - "accuracy": 0.9741666666666666 - } - }, - "auroc": 0.9724399305555556 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 6533, - "fn": 267, - "accuracy": 0.960735294117647 - }, - "0.01": { - "tp": 6367, - "fn": 433, - "accuracy": 0.9363235294117647 - } - }, - "auroc": 0.9575217218137255 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999989583333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99986875 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999338541666667 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999846875 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9982489583333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990479166666667 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999229166666667 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990588541666666 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994908854166666 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9829385416666667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.983121875 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9830302083333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.928984375 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9693927083333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - } - }, - "auroc": 0.9491885416666668 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9559614583333333 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9762572916666667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 781, - "fn": 19, - "accuracy": 0.97625 - }, - "0.01": { - "tp": 769, - "fn": 31, - "accuracy": 0.96125 - } - }, - "auroc": 0.966109375 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.997771875 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.995903125 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9968375 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9984645833333333 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9878302083333333 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9931473958333333 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9981182291666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9918666666666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9949924479166667 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9755583333333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985875 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9870729166666666 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9710885416666667 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9585802083333334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9648343749999999 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9733234375 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9785838541666667 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 785, - "fn": 15, - "accuracy": 0.98125 - } - }, - "auroc": 0.9759536458333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9959135416666667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9803583333333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9881359375000001 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9240177083333334 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9293385416666666 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - }, - "0.01": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - } - }, - "auroc": 0.926678125 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.959965625 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9548484375 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 771, - "fn": 29, - "accuracy": 0.96375 - }, - "0.01": { - "tp": 754, - "fn": 46, - "accuracy": 0.9425 - } - }, - "auroc": 0.95740703125 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994291666666666 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9969406250000001 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9981848958333335 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9975822916666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9736437499999999 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9856130208333334 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985057291666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9852921875 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - } - }, - "auroc": 0.9918989583333333 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9321375 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9321375 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9231114583333334 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9231114583333334 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9276244791666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9276244791666667 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - }, - "0.01": { - "tp": 146, - "fn": 54, - "accuracy": 0.73 - } - }, - "auroc": 0.8778270833333335 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - }, - "0.01": { - "tp": 146, - "fn": 54, - "accuracy": 0.73 - } - }, - "auroc": 0.8778270833333335 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 156, - "fn": 44, - "accuracy": 0.78 - }, - "0.01": { - "tp": 141, - "fn": 59, - "accuracy": 0.705 - } - }, - "auroc": 0.8565572916666667 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 156, - "fn": 44, - "accuracy": 0.78 - }, - "0.01": { - "tp": 141, - "fn": 59, - "accuracy": 0.705 - } - }, - "auroc": 0.8565572916666667 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - }, - "0.01": { - "tp": 287, - "fn": 113, - "accuracy": 0.7175 - } - }, - "auroc": 0.8671921875 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - }, - "0.01": { - "tp": 287, - "fn": 113, - "accuracy": 0.7175 - } - }, - "auroc": 0.8671921875 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992635416666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992635416666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996307291666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996307291666666 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999958333333334 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999958333333334 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9790833333333332 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9790833333333332 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9895395833333334 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9895395833333334 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9713052083333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9713052083333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9504125000000001 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9504125000000001 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9608588541666667 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9608588541666667 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2152, - "fn": 48, - "accuracy": 0.9781818181818182 - }, - "0.01": { - "tp": 2113, - "fn": 87, - "accuracy": 0.9604545454545454 - } - }, - "auroc": 0.9757158143939395 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1196, - "fn": 4, - "accuracy": 0.9966666666666667 - }, - "0.01": { - "tp": 1193, - "fn": 7, - "accuracy": 0.9941666666666666 - } - }, - "auroc": 0.9924633680555556 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3348, - "fn": 52, - "accuracy": 0.9847058823529412 - }, - "0.01": { - "tp": 3306, - "fn": 94, - "accuracy": 0.9723529411764706 - } - }, - "auroc": 0.9816267156862744 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2101, - "fn": 99, - "accuracy": 0.955 - }, - "0.01": { - "tp": 2051, - "fn": 149, - "accuracy": 0.9322727272727273 - } - }, - "auroc": 0.9571284090909092 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1177, - "fn": 23, - "accuracy": 0.9808333333333333 - }, - "0.01": { - "tp": 1157, - "fn": 43, - "accuracy": 0.9641666666666666 - } - }, - "auroc": 0.9695057291666668 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3278, - "fn": 122, - "accuracy": 0.9641176470588235 - }, - "0.01": { - "tp": 3208, - "fn": 192, - "accuracy": 0.9435294117647058 - } - }, - "auroc": 0.9614968749999999 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 4253, - "fn": 147, - "accuracy": 0.9665909090909091 - }, - "0.01": { - "tp": 4164, - "fn": 236, - "accuracy": 0.9463636363636364 - } - }, - "auroc": 0.9664221117424242 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2373, - "fn": 27, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 2350, - "fn": 50, - "accuracy": 0.9791666666666666 - } - }, - "auroc": 0.9809845486111111 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 6626, - "fn": 174, - "accuracy": 0.9744117647058823 - }, - "0.01": { - "tp": 6514, - "fn": 286, - "accuracy": 0.9579411764705882 - } - }, - "auroc": 0.9715617953431372 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980093750000001 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9937479166666666 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9958786458333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9974427083333334 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9862156249999999 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9918291666666668 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9977260416666667 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9899817708333334 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9938539062499999 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.97673125 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 152, - "fn": 48, - "accuracy": 0.76 - } - }, - "auroc": 0.8669625 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - } - }, - "auroc": 0.921846875 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.950090625 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 155, - "fn": 45, - "accuracy": 0.775 - }, - "0.01": { - "tp": 116, - "fn": 84, - "accuracy": 0.58 - } - }, - "auroc": 0.8276447916666666 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - }, - "0.01": { - "tp": 302, - "fn": 98, - "accuracy": 0.755 - } - }, - "auroc": 0.8888677083333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9634109375 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 330, - "fn": 70, - "accuracy": 0.825 - }, - "0.01": { - "tp": 268, - "fn": 132, - "accuracy": 0.67 - } - }, - "auroc": 0.8473036458333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 724, - "fn": 76, - "accuracy": 0.905 - }, - "0.01": { - "tp": 653, - "fn": 147, - "accuracy": 0.81625 - } - }, - "auroc": 0.9053572916666668 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9660875 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.9190479166666666 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9425677083333333 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9682895833333334 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.91404375 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9411666666666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9671885416666668 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 345, - "fn": 55, - "accuracy": 0.8625 - } - }, - "auroc": 0.9165458333333334 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 771, - "fn": 29, - "accuracy": 0.96375 - }, - "0.01": { - "tp": 732, - "fn": 68, - "accuracy": 0.915 - } - }, - "auroc": 0.9418671875 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9593447916666666 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.967653125 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9634989583333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9490343750000001 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 157, - "fn": 43, - "accuracy": 0.785 - } - }, - "auroc": 0.8787916666666666 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 346, - "fn": 54, - "accuracy": 0.865 - } - }, - "auroc": 0.9139130208333334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9541895833333334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 349, - "fn": 51, - "accuracy": 0.8725 - } - }, - "auroc": 0.9232223958333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 774, - "fn": 26, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 730, - "fn": 70, - "accuracy": 0.9125 - } - }, - "auroc": 0.9387059895833333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9704395833333335 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - }, - "0.01": { - "tp": 140, - "fn": 60, - "accuracy": 0.7 - } - }, - "auroc": 0.8524072916666665 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - }, - "0.01": { - "tp": 337, - "fn": 63, - "accuracy": 0.8425 - } - }, - "auroc": 0.9114234375000001 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9094822916666666 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 135, - "fn": 65, - "accuracy": 0.675 - }, - "0.01": { - "tp": 107, - "fn": 93, - "accuracy": 0.535 - } - }, - "auroc": 0.7951604166666667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 323, - "fn": 77, - "accuracy": 0.8075 - }, - "0.01": { - "tp": 275, - "fn": 125, - "accuracy": 0.6875 - } - }, - "auroc": 0.8523213541666668 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9399609375 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 302, - "fn": 98, - "accuracy": 0.755 - }, - "0.01": { - "tp": 247, - "fn": 153, - "accuracy": 0.6175 - } - }, - "auroc": 0.8237838541666668 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 689, - "fn": 111, - "accuracy": 0.86125 - }, - "0.01": { - "tp": 612, - "fn": 188, - "accuracy": 0.765 - } - }, - "auroc": 0.8818723958333332 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9877614583333334 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.967846875 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9778041666666668 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.982359375 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.95033125 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9663453125 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9850604166666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9590890625000001 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 778, - "fn": 22, - "accuracy": 0.9725 - } - }, - "auroc": 0.9720747395833333 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.88825 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.88825 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 157, - "fn": 43, - "accuracy": 0.785 - } - }, - "auroc": 0.89100625 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 157, - "fn": 43, - "accuracy": 0.785 - } - }, - "auroc": 0.89100625 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - }, - "0.01": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - } - }, - "auroc": 0.889628125 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - }, - "0.01": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - } - }, - "auroc": 0.889628125 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - }, - "0.01": { - "tp": 141, - "fn": 59, - "accuracy": 0.705 - } - }, - "auroc": 0.8599854166666666 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - }, - "0.01": { - "tp": 141, - "fn": 59, - "accuracy": 0.705 - } - }, - "auroc": 0.8599854166666666 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 154, - "fn": 46, - "accuracy": 0.77 - }, - "0.01": { - "tp": 131, - "fn": 69, - "accuracy": 0.655 - } - }, - "auroc": 0.840278125 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 154, - "fn": 46, - "accuracy": 0.77 - }, - "0.01": { - "tp": 131, - "fn": 69, - "accuracy": 0.655 - } - }, - "auroc": 0.840278125 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 323, - "fn": 77, - "accuracy": 0.8075 - }, - "0.01": { - "tp": 272, - "fn": 128, - "accuracy": 0.68 - } - }, - "auroc": 0.8501317708333332 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 323, - "fn": 77, - "accuracy": 0.8075 - }, - "0.01": { - "tp": 272, - "fn": 128, - "accuracy": 0.68 - } - }, - "auroc": 0.8501317708333332 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9792489583333335 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9792489583333335 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9780583333333334 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9780583333333334 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9786536458333334 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9786536458333334 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.983025 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.983025 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.949025 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.949025 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.966025 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.966025 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9505822916666666 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9505822916666666 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9283708333333334 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9283708333333334 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.9394765625000001 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.9394765625000001 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2125, - "fn": 75, - "accuracy": 0.9659090909090909 - }, - "0.01": { - "tp": 2055, - "fn": 145, - "accuracy": 0.9340909090909091 - } - }, - "auroc": 0.9563150568181817 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1125, - "fn": 75, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 1052, - "fn": 148, - "accuracy": 0.8766666666666667 - } - }, - "auroc": 0.9279442708333332 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3250, - "fn": 150, - "accuracy": 0.9558823529411765 - }, - "0.01": { - "tp": 3107, - "fn": 293, - "accuracy": 0.9138235294117647 - } - }, - "auroc": 0.9463018382352942 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2088, - "fn": 112, - "accuracy": 0.9490909090909091 - }, - "0.01": { - "tp": 1980, - "fn": 220, - "accuracy": 0.9 - } - }, - "auroc": 0.9403125 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1058, - "fn": 142, - "accuracy": 0.8816666666666667 - }, - "0.01": { - "tp": 939, - "fn": 261, - "accuracy": 0.7825 - } - }, - "auroc": 0.8920312500000002 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3146, - "fn": 254, - "accuracy": 0.9252941176470588 - }, - "0.01": { - "tp": 2919, - "fn": 481, - "accuracy": 0.8585294117647059 - } - }, - "auroc": 0.9232720588235295 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 4213, - "fn": 187, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 4035, - "fn": 365, - "accuracy": 0.9170454545454545 - } - }, - "auroc": 0.948313778409091 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2183, - "fn": 217, - "accuracy": 0.9095833333333333 - }, - "0.01": { - "tp": 1991, - "fn": 409, - "accuracy": 0.8295833333333333 - } - }, - "auroc": 0.9099877604166665 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 6396, - "fn": 404, - "accuracy": 0.9405882352941176 - }, - "0.01": { - "tp": 6026, - "fn": 774, - "accuracy": 0.8861764705882353 - } - }, - "auroc": 0.9347869485294118 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99988125 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999940625 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999815625 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9981447916666666 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9989802083333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999078125 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9990130208333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9994604166666667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9818864583333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9835083333333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9826973958333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9130406249999998 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9685552083333333 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9407979166666667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - } - }, - "auroc": 0.9474635416666668 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9760317708333335 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 777, - "fn": 23, - "accuracy": 0.97125 - }, - "0.01": { - "tp": 763, - "fn": 37, - "accuracy": 0.95375 - } - }, - "auroc": 0.96174765625 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.997675 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9960562499999999 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.996865625 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983822916666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9878791666666666 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9931307291666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9980286458333333 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9919677083333334 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9949981770833334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9752104166666667 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9977645833333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9864875 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9642177083333334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9468447916666667 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.95553125 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9697140625 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9723046875 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 782, - "fn": 18, - "accuracy": 0.9775 - } - }, - "auroc": 0.9710093750000001 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9958552083333334 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.974134375 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9849947916666666 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.912728125 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.924340625 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9185343750000001 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9542916666666668 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9492375 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 769, - "fn": 31, - "accuracy": 0.96125 - }, - "0.01": { - "tp": 751, - "fn": 49, - "accuracy": 0.93875 - } - }, - "auroc": 0.9517645833333335 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994447916666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9969697916666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9982072916666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9969489583333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9711510416666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9840500000000001 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998196875 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9840604166666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - } - }, - "auroc": 0.9911286458333333 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9320541666666666 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9320541666666666 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9202822916666666 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9202822916666666 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9261682291666666 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9261682291666666 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - }, - "0.01": { - "tp": 136, - "fn": 64, - "accuracy": 0.68 - } - }, - "auroc": 0.8626510416666666 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - }, - "0.01": { - "tp": 136, - "fn": 64, - "accuracy": 0.68 - } - }, - "auroc": 0.8626510416666666 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 133, - "fn": 67, - "accuracy": 0.665 - } - }, - "auroc": 0.8349822916666667 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 133, - "fn": 67, - "accuracy": 0.665 - } - }, - "auroc": 0.8349822916666667 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 314, - "fn": 86, - "accuracy": 0.785 - }, - "0.01": { - "tp": 269, - "fn": 131, - "accuracy": 0.6725 - } - }, - "auroc": 0.8488166666666667 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 314, - "fn": 86, - "accuracy": 0.785 - }, - "0.01": { - "tp": 269, - "fn": 131, - "accuracy": 0.6725 - } - }, - "auroc": 0.8488166666666667 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990479166666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990479166666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995151041666667 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995151041666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666666 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666666 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9772958333333333 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9772958333333333 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9886468750000001 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9886468750000001 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9641708333333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9641708333333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9460489583333334 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9460489583333334 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9551098958333334 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9551098958333334 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2141, - "fn": 59, - "accuracy": 0.9731818181818181 - }, - "0.01": { - "tp": 2102, - "fn": 98, - "accuracy": 0.9554545454545454 - } - }, - "auroc": 0.9735389204545455 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1196, - "fn": 4, - "accuracy": 0.9966666666666667 - }, - "0.01": { - "tp": 1193, - "fn": 7, - "accuracy": 0.9941666666666666 - } - }, - "auroc": 0.9913857638888889 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3337, - "fn": 63, - "accuracy": 0.9814705882352941 - }, - "0.01": { - "tp": 3295, - "fn": 105, - "accuracy": 0.9691176470588235 - } - }, - "auroc": 0.979837806372549 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2090, - "fn": 110, - "accuracy": 0.95 - }, - "0.01": { - "tp": 2032, - "fn": 168, - "accuracy": 0.9236363636363636 - } - }, - "auroc": 0.9511627840909092 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1176, - "fn": 24, - "accuracy": 0.98 - }, - "0.01": { - "tp": 1153, - "fn": 47, - "accuracy": 0.9608333333333333 - } - }, - "auroc": 0.9661526041666666 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3266, - "fn": 134, - "accuracy": 0.9605882352941176 - }, - "0.01": { - "tp": 3185, - "fn": 215, - "accuracy": 0.9367647058823529 - } - }, - "auroc": 0.9564533088235294 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 4231, - "fn": 169, - "accuracy": 0.961590909090909 - }, - "0.01": { - "tp": 4134, - "fn": 266, - "accuracy": 0.9395454545454546 - } - }, - "auroc": 0.9623508522727273 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2372, - "fn": 28, - "accuracy": 0.9883333333333333 - }, - "0.01": { - "tp": 2346, - "fn": 54, - "accuracy": 0.9775 - } - }, - "auroc": 0.9787691840277777 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 6603, - "fn": 197, - "accuracy": 0.9710294117647059 - }, - "0.01": { - "tp": 6480, - "fn": 320, - "accuracy": 0.9529411764705882 - } - }, - "auroc": 0.9681455575980391 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998802083333334 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999401041666667 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99985 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.99819375 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.999021875 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999250000000001 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9990369791666667 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9994809895833332 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9798572916666666 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9833302083333333 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9815937499999999 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.90664375 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.969096875 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9378703125 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9432505208333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9762135416666666 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 775, - "fn": 25, - "accuracy": 0.96875 - }, - "0.01": { - "tp": 765, - "fn": 35, - "accuracy": 0.95625 - } - }, - "auroc": 0.9597320312500001 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9976499999999999 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9960104166666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9968302083333334 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9984708333333333 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.98809375 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9932822916666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9980604166666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9920520833333333 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.99505625 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9754833333333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983385416666667 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9869109375 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9625354166666666 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9448177083333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9536765624999999 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.969009375 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9715781250000001 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 782, - "fn": 18, - "accuracy": 0.9775 - } - }, - "auroc": 0.97029375 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9952885416666666 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.97391875 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9846036458333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9053281249999999 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9217489583333334 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9135385416666667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9503083333333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9478338541666667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 769, - "fn": 31, - "accuracy": 0.96125 - }, - "0.01": { - "tp": 750, - "fn": 50, - "accuracy": 0.9375 - } - }, - "auroc": 0.9490710937499999 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994458333333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9973916666666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9984187499999999 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9976166666666666 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9793979166666666 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9885072916666666 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99853125 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9883947916666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - } - }, - "auroc": 0.9934630208333334 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.931759375 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.931759375 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.9190166666666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.9190166666666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9253880208333334 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9253880208333334 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - }, - "0.01": { - "tp": 136, - "fn": 64, - "accuracy": 0.68 - } - }, - "auroc": 0.8623479166666665 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - }, - "0.01": { - "tp": 136, - "fn": 64, - "accuracy": 0.68 - } - }, - "auroc": 0.8623479166666665 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 149, - "fn": 51, - "accuracy": 0.745 - }, - "0.01": { - "tp": 133, - "fn": 67, - "accuracy": 0.665 - } - }, - "auroc": 0.8348729166666665 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 149, - "fn": 51, - "accuracy": 0.745 - }, - "0.01": { - "tp": 133, - "fn": 67, - "accuracy": 0.665 - } - }, - "auroc": 0.8348729166666665 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 313, - "fn": 87, - "accuracy": 0.7825 - }, - "0.01": { - "tp": 269, - "fn": 131, - "accuracy": 0.6725 - } - }, - "auroc": 0.8486104166666667 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 313, - "fn": 87, - "accuracy": 0.7825 - }, - "0.01": { - "tp": 269, - "fn": 131, - "accuracy": 0.6725 - } - }, - "auroc": 0.8486104166666667 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999760416666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999760416666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9991458333333333 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9991458333333333 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995609375000001 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995609375000001 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666666 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666666 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9771656249999999 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9771656249999999 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9885817708333333 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9885817708333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9648135416666667 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9648135416666667 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9461416666666668 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9461416666666668 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9554776041666668 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9554776041666668 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2143, - "fn": 57, - "accuracy": 0.9740909090909091 - }, - "0.01": { - "tp": 2101, - "fn": 99, - "accuracy": 0.955 - } - }, - "auroc": 0.973329071969697 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1194, - "fn": 6, - "accuracy": 0.995 - } - }, - "auroc": 0.991478298611111 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3340, - "fn": 60, - "accuracy": 0.9823529411764705 - }, - "0.01": { - "tp": 3295, - "fn": 105, - "accuracy": 0.9691176470588235 - } - }, - "auroc": 0.9797346813725489 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2087, - "fn": 113, - "accuracy": 0.9486363636363636 - }, - "0.01": { - "tp": 2033, - "fn": 167, - "accuracy": 0.9240909090909091 - } - }, - "auroc": 0.9497079545454545 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1176, - "fn": 24, - "accuracy": 0.98 - }, - "0.01": { - "tp": 1151, - "fn": 49, - "accuracy": 0.9591666666666666 - } - }, - "auroc": 0.9668914930555554 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3263, - "fn": 137, - "accuracy": 0.9597058823529412 - }, - "0.01": { - "tp": 3184, - "fn": 216, - "accuracy": 0.9364705882352942 - } - }, - "auroc": 0.9557727328431374 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 4230, - "fn": 170, - "accuracy": 0.9613636363636363 - }, - "0.01": { - "tp": 4134, - "fn": 266, - "accuracy": 0.9395454545454546 - } - }, - "auroc": 0.9615185132575758 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2373, - "fn": 27, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 2345, - "fn": 55, - "accuracy": 0.9770833333333333 - } - }, - "auroc": 0.9791848958333333 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 6603, - "fn": 197, - "accuracy": 0.9710294117647059 - }, - "0.01": { - "tp": 6479, - "fn": 321, - "accuracy": 0.9527941176470588 - } - }, - "auroc": 0.9677537071078433 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9586020833333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9423020833333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9504520833333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.959346875 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9405593750000001 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.949953125 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9589744791666667 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9414307291666667 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9502026041666667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9953395833333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9832260416666667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9892828125 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.958446875 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9834614583333332 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9709541666666667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9768932291666667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9833437500000001 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - } - }, - "auroc": 0.9801184895833334 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9614385416666668 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9567510416666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9590947916666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.963409375 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9698666666666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9666380208333334 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9624239583333334 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9633088541666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.96286640625 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9800166666666666 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9811677083333332 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9805921875 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9841645833333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9735895833333335 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9788770833333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.982090625 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9773786458333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - } - }, - "auroc": 0.9797346354166667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9966427083333332 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9819770833333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9893098958333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9740166666666668 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9741302083333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9740734375 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9853296875 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9780536458333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9816916666666666 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.97211875 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9496739583333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9608963541666666 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.960553125 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9513625000000001 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9559578125 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9663359375 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9505182291666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9584270833333333 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9775604166666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9775604166666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9763822916666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9763822916666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9769713541666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9769713541666667 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9761572916666668 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9761572916666668 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9680562500000001 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9680562500000001 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9721067708333334 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9721067708333334 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9637916666666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9637916666666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9587531250000001 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9587531250000001 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9612723958333333 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9612723958333333 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.939540625 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.939540625 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9606979166666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9606979166666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9501192708333333 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9501192708333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9742374999999999 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9742374999999999 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9707614583333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9707614583333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9724994791666668 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9724994791666668 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2199, - "fn": 1, - "accuracy": 0.9995454545454545 - }, - "0.01": { - "tp": 2193, - "fn": 7, - "accuracy": 0.9968181818181818 - } - }, - "auroc": 0.9723132575757576 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1199, - "fn": 1, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 1199, - "fn": 1, - "accuracy": 0.9991666666666666 - } - }, - "auroc": 0.9658496527777778 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3398, - "fn": 2, - "accuracy": 0.9994117647058823 - }, - "0.01": { - "tp": 3392, - "fn": 8, - "accuracy": 0.9976470588235294 - } - }, - "auroc": 0.9700319852941177 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2196, - "fn": 4, - "accuracy": 0.9981818181818182 - }, - "0.01": { - "tp": 2191, - "fn": 9, - "accuracy": 0.9959090909090909 - } - }, - "auroc": 0.9667807765151515 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1195, - "fn": 5, - "accuracy": 0.9958333333333333 - }, - "0.01": { - "tp": 1186, - "fn": 14, - "accuracy": 0.9883333333333333 - } - }, - "auroc": 0.9654949652777778 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3391, - "fn": 9, - "accuracy": 0.9973529411764706 - }, - "0.01": { - "tp": 3377, - "fn": 23, - "accuracy": 0.9932352941176471 - } - }, - "auroc": 0.9663269607843138 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 4395, - "fn": 5, - "accuracy": 0.9988636363636364 - }, - "0.01": { - "tp": 4384, - "fn": 16, - "accuracy": 0.9963636363636363 - } - }, - "auroc": 0.9695470170454545 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2394, - "fn": 6, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 2385, - "fn": 15, - "accuracy": 0.99375 - } - }, - "auroc": 0.9656723090277778 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 6789, - "fn": 11, - "accuracy": 0.9983823529411765 - }, - "0.01": { - "tp": 6769, - "fn": 31, - "accuracy": 0.9954411764705883 - } - }, - "auroc": 0.9681794730392157 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.997284375 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9959239583333334 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9966041666666667 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9970531250000001 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9934479166666667 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9952505208333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9971687499999999 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9946859375000001 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.99592734375 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9669531250000001 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9824375000000001 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9746953125 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.91126875 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.968965625 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9401171875000001 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9391109375000001 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9757015625 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 786, - "fn": 14, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 776, - "fn": 24, - "accuracy": 0.97 - } - }, - "auroc": 0.95740625 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9931041666666666 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9939375 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9935208333333334 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9929947916666666 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9873760416666666 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9901854166666666 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9930494791666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9906567708333334 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.991853125 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9636135416666667 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9894833333333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9765484375 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9436906250000001 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9542302083333334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9489604166666668 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9536520833333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9718567708333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 783, - "fn": 17, - "accuracy": 0.97875 - } - }, - "auroc": 0.9627544270833334 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9910020833333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9645010416666666 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9777515625 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.8933208333333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9295260416666666 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9114234375 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9421614583333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9470135416666667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 773, - "fn": 27, - "accuracy": 0.96625 - }, - "0.01": { - "tp": 754, - "fn": 46, - "accuracy": 0.9425 - } - }, - "auroc": 0.9445875000000001 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9887979166666666 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.989046875 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9889223958333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9846874999999999 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9697052083333334 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9771963541666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9867427083333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9793760416666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - } - }, - "auroc": 0.9830593750000001 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.8658697916666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.8658697916666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.8579104166666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.8579104166666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - }, - "0.01": { - "tp": 323, - "fn": 77, - "accuracy": 0.8075 - } - }, - "auroc": 0.8618901041666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - }, - "0.01": { - "tp": 323, - "fn": 77, - "accuracy": 0.8075 - } - }, - "auroc": 0.8618901041666667 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 149, - "fn": 51, - "accuracy": 0.745 - }, - "0.01": { - "tp": 137, - "fn": 63, - "accuracy": 0.685 - } - }, - "auroc": 0.8257479166666667 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 149, - "fn": 51, - "accuracy": 0.745 - }, - "0.01": { - "tp": 137, - "fn": 63, - "accuracy": 0.685 - } - }, - "auroc": 0.8257479166666667 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 145, - "fn": 55, - "accuracy": 0.725 - }, - "0.01": { - "tp": 129, - "fn": 71, - "accuracy": 0.645 - } - }, - "auroc": 0.7907510416666665 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 145, - "fn": 55, - "accuracy": 0.725 - }, - "0.01": { - "tp": 129, - "fn": 71, - "accuracy": 0.645 - } - }, - "auroc": 0.7907510416666665 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 294, - "fn": 106, - "accuracy": 0.735 - }, - "0.01": { - "tp": 266, - "fn": 134, - "accuracy": 0.665 - } - }, - "auroc": 0.8082494791666668 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 294, - "fn": 106, - "accuracy": 0.735 - }, - "0.01": { - "tp": 266, - "fn": 134, - "accuracy": 0.665 - } - }, - "auroc": 0.8082494791666668 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9946927083333335 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9946927083333335 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9914020833333332 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9914020833333332 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9930473958333333 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9930473958333333 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9979020833333332 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9979020833333332 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9691802083333333 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9691802083333333 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9835411458333334 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9835411458333334 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9410604166666667 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9410604166666667 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9283177083333334 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9283177083333334 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9346890625000001 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9346890625000001 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2116, - "fn": 84, - "accuracy": 0.9618181818181818 - }, - "0.01": { - "tp": 2084, - "fn": 116, - "accuracy": 0.9472727272727273 - } - }, - "auroc": 0.9569116477272728 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1194, - "fn": 6, - "accuracy": 0.995 - } - }, - "auroc": 0.9858883680555555 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3313, - "fn": 87, - "accuracy": 0.9744117647058823 - }, - "0.01": { - "tp": 3278, - "fn": 122, - "accuracy": 0.9641176470588235 - } - }, - "auroc": 0.967138725490196 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2085, - "fn": 115, - "accuracy": 0.9477272727272728 - }, - "0.01": { - "tp": 2026, - "fn": 174, - "accuracy": 0.9209090909090909 - } - }, - "auroc": 0.9327797348484849 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1175, - "fn": 25, - "accuracy": 0.9791666666666666 - }, - "0.01": { - "tp": 1154, - "fn": 46, - "accuracy": 0.9616666666666667 - } - }, - "auroc": 0.9672085069444445 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3260, - "fn": 140, - "accuracy": 0.9588235294117647 - }, - "0.01": { - "tp": 3180, - "fn": 220, - "accuracy": 0.9352941176470588 - } - }, - "auroc": 0.9449310661764705 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 4201, - "fn": 199, - "accuracy": 0.9547727272727272 - }, - "0.01": { - "tp": 4110, - "fn": 290, - "accuracy": 0.9340909090909091 - } - }, - "auroc": 0.9448456912878788 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2372, - "fn": 28, - "accuracy": 0.9883333333333333 - }, - "0.01": { - "tp": 2348, - "fn": 52, - "accuracy": 0.9783333333333334 - } - }, - "auroc": 0.9765484375 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 6573, - "fn": 227, - "accuracy": 0.9666176470588236 - }, - "0.01": { - "tp": 6458, - "fn": 342, - "accuracy": 0.9497058823529412 - } - }, - "auroc": 0.9560348958333333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999968749999999 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999825 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999109374999999 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999846875 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9980625 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9989546874999999 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999218750000001 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.99894375 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9994328125 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.982534375 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9835875000000001 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9830609374999999 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9261177083333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9697708333333332 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9479442708333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9543260416666666 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9766791666666668 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 782, - "fn": 18, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 773, - "fn": 27, - "accuracy": 0.96625 - } - }, - "auroc": 0.9655026041666668 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9973083333333332 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9956218749999999 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9964651041666666 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985937500000001 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9885833333333333 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9935885416666668 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9979510416666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9921026041666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9950268229166667 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9754854166666667 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983802083333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9869328125 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.97069375 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9506708333333334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9606822916666667 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9730895833333334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9745255208333333 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 784, - "fn": 16, - "accuracy": 0.98 - } - }, - "auroc": 0.9738075520833334 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9955218750000001 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9775114583333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9865166666666666 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9250135416666667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9274875 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9262505208333334 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9602677083333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9524994791666666 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 772, - "fn": 28, - "accuracy": 0.965 - }, - "0.01": { - "tp": 753, - "fn": 47, - "accuracy": 0.94125 - } - }, - "auroc": 0.95638359375 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994760416666666 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9965833333333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9980296875 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9973302083333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9731614583333335 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9852458333333334 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9984031250000001 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9848723958333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - } - }, - "auroc": 0.9916377604166666 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9390916666666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9390916666666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9266812499999999 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9266812499999999 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.9328864583333335 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.9328864583333335 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - }, - "0.01": { - "tp": 141, - "fn": 59, - "accuracy": 0.705 - } - }, - "auroc": 0.8705479166666668 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - }, - "0.01": { - "tp": 141, - "fn": 59, - "accuracy": 0.705 - } - }, - "auroc": 0.8705479166666668 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 154, - "fn": 46, - "accuracy": 0.77 - }, - "0.01": { - "tp": 136, - "fn": 64, - "accuracy": 0.68 - } - }, - "auroc": 0.8414875 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 154, - "fn": 46, - "accuracy": 0.77 - }, - "0.01": { - "tp": 136, - "fn": 64, - "accuracy": 0.68 - } - }, - "auroc": 0.8414875 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 319, - "fn": 81, - "accuracy": 0.7975 - }, - "0.01": { - "tp": 277, - "fn": 123, - "accuracy": 0.6925 - } - }, - "auroc": 0.8560177083333333 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 319, - "fn": 81, - "accuracy": 0.7975 - }, - "0.01": { - "tp": 277, - "fn": 123, - "accuracy": 0.6925 - } - }, - "auroc": 0.8560177083333333 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999666666666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999666666666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989989583333334 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989989583333334 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994828125 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994828125 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999270833333334 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999270833333334 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.979071875 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.979071875 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9894994791666666 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9894994791666666 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.96721875 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.96721875 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9503166666666667 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9503166666666667 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9587677083333335 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9587677083333335 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2147, - "fn": 53, - "accuracy": 0.975909090909091 - }, - "0.01": { - "tp": 2108, - "fn": 92, - "accuracy": 0.9581818181818181 - } - }, - "auroc": 0.9751886363636364 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1193, - "fn": 7, - "accuracy": 0.9941666666666666 - } - }, - "auroc": 0.9919182291666666 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3344, - "fn": 56, - "accuracy": 0.9835294117647059 - }, - "0.01": { - "tp": 3301, - "fn": 99, - "accuracy": 0.9708823529411764 - } - }, - "auroc": 0.9810931985294117 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2105, - "fn": 95, - "accuracy": 0.9568181818181818 - }, - "0.01": { - "tp": 2052, - "fn": 148, - "accuracy": 0.9327272727272727 - } - }, - "auroc": 0.9558320075757576 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1177, - "fn": 23, - "accuracy": 0.9808333333333333 - }, - "0.01": { - "tp": 1156, - "fn": 44, - "accuracy": 0.9633333333333334 - } - }, - "auroc": 0.967956076388889 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3282, - "fn": 118, - "accuracy": 0.9652941176470589 - }, - "0.01": { - "tp": 3208, - "fn": 192, - "accuracy": 0.9435294117647058 - } - }, - "auroc": 0.9601110906862744 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 4252, - "fn": 148, - "accuracy": 0.9663636363636363 - }, - "0.01": { - "tp": 4160, - "fn": 240, - "accuracy": 0.9454545454545454 - } - }, - "auroc": 0.965510321969697 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2374, - "fn": 26, - "accuracy": 0.9891666666666666 - }, - "0.01": { - "tp": 2349, - "fn": 51, - "accuracy": 0.97875 - } - }, - "auroc": 0.9799371527777778 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 6626, - "fn": 174, - "accuracy": 0.9744117647058823 - }, - "0.01": { - "tp": 6509, - "fn": 291, - "accuracy": 0.9572058823529411 - } - }, - "auroc": 0.9706021446078432 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998906249999999 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999453125 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998468749999999 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9981510416666668 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9989989583333334 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999234375 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9990208333333335 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9994721354166667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9823104166666665 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9834187500000001 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9828645833333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9140593750000001 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9689864583333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9415229166666667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9481848958333334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9762026041666667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 777, - "fn": 23, - "accuracy": 0.97125 - }, - "0.01": { - "tp": 766, - "fn": 34, - "accuracy": 0.9575 - } - }, - "auroc": 0.9621937500000001 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9976750000000001 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.996028125 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9968515625 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985020833333333 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9878739583333334 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9931880208333332 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9980885416666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9919510416666667 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9950197916666668 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9753906250000001 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998325 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9868578125 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.963790625 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9453947916666666 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9545927083333334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9695906250000002 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9718598958333332 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 783, - "fn": 17, - "accuracy": 0.97875 - } - }, - "auroc": 0.9707252604166667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9959072916666667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.974028125 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9849677083333334 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9127208333333332 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.924065625 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.9183932291666668 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9543140625 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9490468750000001 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 769, - "fn": 31, - "accuracy": 0.96125 - }, - "0.01": { - "tp": 752, - "fn": 48, - "accuracy": 0.94 - } - }, - "auroc": 0.95168046875 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994447916666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9969697916666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9982072916666667 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9971687499999999 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.971071875 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9841203125000001 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983067708333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9840208333333333 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - } - }, - "auroc": 0.9911638020833333 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9331260416666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9331260416666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9207979166666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9207979166666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9269619791666668 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9269619791666668 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - }, - "0.01": { - "tp": 137, - "fn": 63, - "accuracy": 0.685 - } - }, - "auroc": 0.863309375 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - }, - "0.01": { - "tp": 137, - "fn": 63, - "accuracy": 0.685 - } - }, - "auroc": 0.863309375 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 152, - "fn": 48, - "accuracy": 0.76 - }, - "0.01": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - } - }, - "auroc": 0.8356281249999999 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 152, - "fn": 48, - "accuracy": 0.76 - }, - "0.01": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - } - }, - "auroc": 0.8356281249999999 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 316, - "fn": 84, - "accuracy": 0.79 - }, - "0.01": { - "tp": 271, - "fn": 129, - "accuracy": 0.6775 - } - }, - "auroc": 0.84946875 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 316, - "fn": 84, - "accuracy": 0.79 - }, - "0.01": { - "tp": 271, - "fn": 129, - "accuracy": 0.6775 - } - }, - "auroc": 0.84946875 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9991708333333333 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9991708333333333 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995765624999999 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995765624999999 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9773729166666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9773729166666667 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9886854166666668 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9886854166666668 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9651770833333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9651770833333333 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.945515625 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.945515625 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9553463541666667 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9553463541666667 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2144, - "fn": 56, - "accuracy": 0.9745454545454545 - }, - "0.01": { - "tp": 2103, - "fn": 97, - "accuracy": 0.9559090909090909 - } - }, - "auroc": 0.9738473484848484 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1194, - "fn": 6, - "accuracy": 0.995 - } - }, - "auroc": 0.9914434027777779 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3341, - "fn": 59, - "accuracy": 0.9826470588235294 - }, - "0.01": { - "tp": 3297, - "fn": 103, - "accuracy": 0.9697058823529412 - } - }, - "auroc": 0.9800577205882353 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2092, - "fn": 108, - "accuracy": 0.9509090909090909 - }, - "0.01": { - "tp": 2035, - "fn": 165, - "accuracy": 0.925 - } - }, - "auroc": 0.9513249053030302 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1176, - "fn": 24, - "accuracy": 0.98 - }, - "0.01": { - "tp": 1154, - "fn": 46, - "accuracy": 0.9616666666666667 - } - }, - "auroc": 0.9659239583333334 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3268, - "fn": 132, - "accuracy": 0.9611764705882353 - }, - "0.01": { - "tp": 3189, - "fn": 211, - "accuracy": 0.9379411764705883 - } - }, - "auroc": 0.9564775122549021 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 4236, - "fn": 164, - "accuracy": 0.9627272727272728 - }, - "0.01": { - "tp": 4138, - "fn": 262, - "accuracy": 0.9404545454545454 - } - }, - "auroc": 0.9625861268939393 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2373, - "fn": 27, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 2348, - "fn": 52, - "accuracy": 0.9783333333333334 - } - }, - "auroc": 0.9786836805555555 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 6609, - "fn": 191, - "accuracy": 0.9719117647058824 - }, - "0.01": { - "tp": 6486, - "fn": 314, - "accuracy": 0.9538235294117647 - } - }, - "auroc": 0.9682676164215686 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.996140451388889 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9940535590277778 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9950970052083333 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9960291666666667 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2391, - "fn": 9, - "accuracy": 0.99625 - } - }, - "auroc": 0.9917428819444445 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4791, - "fn": 9, - "accuracy": 0.998125 - } - }, - "auroc": 0.9938860243055556 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9960848090277779 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4791, - "fn": 9, - "accuracy": 0.998125 - } - }, - "auroc": 0.9928982204861112 - }, - { - "domain": "abstracts", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 9591, - "fn": 9, - "accuracy": 0.9990625 - } - }, - "auroc": 0.9944915147569444 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9826030381944444 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2353, - "fn": 47, - "accuracy": 0.9804166666666667 - }, - "0.01": { - "tp": 2320, - "fn": 80, - "accuracy": 0.9666666666666667 - } - }, - "auroc": 0.9734059027777778 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4752, - "fn": 48, - "accuracy": 0.99 - }, - "0.01": { - "tp": 4719, - "fn": 81, - "accuracy": 0.983125 - } - }, - "auroc": 0.978004470486111 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2248, - "fn": 152, - "accuracy": 0.9366666666666666 - }, - "0.01": { - "tp": 2182, - "fn": 218, - "accuracy": 0.9091666666666667 - } - }, - "auroc": 0.9254650173611111 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2325, - "fn": 75, - "accuracy": 0.96875 - }, - "0.01": { - "tp": 2238, - "fn": 162, - "accuracy": 0.9325 - } - }, - "auroc": 0.9581336805555556 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4573, - "fn": 227, - "accuracy": 0.9527083333333334 - }, - "0.01": { - "tp": 4420, - "fn": 380, - "accuracy": 0.9208333333333333 - } - }, - "auroc": 0.9417993489583334 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4647, - "fn": 153, - "accuracy": 0.968125 - }, - "0.01": { - "tp": 4581, - "fn": 219, - "accuracy": 0.954375 - } - }, - "auroc": 0.9540340277777778 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4678, - "fn": 122, - "accuracy": 0.9745833333333334 - }, - "0.01": { - "tp": 4558, - "fn": 242, - "accuracy": 0.9495833333333333 - } - }, - "auroc": 0.9657697916666667 - }, - { - "domain": "abstracts", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9325, - "fn": 275, - "accuracy": 0.9713541666666666 - }, - "0.01": { - "tp": 9139, - "fn": 461, - "accuracy": 0.9519791666666667 - } - }, - "auroc": 0.9599019097222222 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2387, - "fn": 13, - "accuracy": 0.9945833333333334 - }, - "0.01": { - "tp": 2380, - "fn": 20, - "accuracy": 0.9916666666666667 - } - }, - "auroc": 0.9908018229166666 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2388, - "fn": 12, - "accuracy": 0.995 - }, - "0.01": { - "tp": 2373, - "fn": 27, - "accuracy": 0.98875 - } - }, - "auroc": 0.9853022569444445 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4775, - "fn": 25, - "accuracy": 0.9947916666666666 - }, - "0.01": { - "tp": 4753, - "fn": 47, - "accuracy": 0.9902083333333334 - } - }, - "auroc": 0.9880520399305557 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2397, - "fn": 3, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 2395, - "fn": 5, - "accuracy": 0.9979166666666667 - } - }, - "auroc": 0.9919368055555555 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2388, - "fn": 12, - "accuracy": 0.995 - }, - "0.01": { - "tp": 2359, - "fn": 41, - "accuracy": 0.9829166666666667 - } - }, - "auroc": 0.97917421875 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4785, - "fn": 15, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 4754, - "fn": 46, - "accuracy": 0.9904166666666666 - } - }, - "auroc": 0.9855555121527777 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4784, - "fn": 16, - "accuracy": 0.9966666666666667 - }, - "0.01": { - "tp": 4775, - "fn": 25, - "accuracy": 0.9947916666666666 - } - }, - "auroc": 0.9913693142361111 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4776, - "fn": 24, - "accuracy": 0.995 - }, - "0.01": { - "tp": 4732, - "fn": 68, - "accuracy": 0.9858333333333333 - } - }, - "auroc": 0.9822382378472223 - }, - { - "domain": "abstracts", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9560, - "fn": 40, - "accuracy": 0.9958333333333333 - }, - "0.01": { - "tp": 9507, - "fn": 93, - "accuracy": 0.9903125 - } - }, - "auroc": 0.9868037760416666 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 2362, - "fn": 38, - "accuracy": 0.9841666666666666 - } - }, - "auroc": 0.9747238715277778 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2397, - "fn": 3, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 2391, - "fn": 9, - "accuracy": 0.99625 - } - }, - "auroc": 0.993131857638889 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4795, - "fn": 5, - "accuracy": 0.9989583333333333 - }, - "0.01": { - "tp": 4753, - "fn": 47, - "accuracy": 0.9902083333333334 - } - }, - "auroc": 0.9839278645833334 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2385, - "fn": 15, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 2348, - "fn": 52, - "accuracy": 0.9783333333333334 - } - }, - "auroc": 0.9648788194444443 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2335, - "fn": 65, - "accuracy": 0.9729166666666667 - }, - "0.01": { - "tp": 2261, - "fn": 139, - "accuracy": 0.9420833333333334 - } - }, - "auroc": 0.9477883680555556 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4720, - "fn": 80, - "accuracy": 0.9833333333333333 - }, - "0.01": { - "tp": 4609, - "fn": 191, - "accuracy": 0.9602083333333333 - } - }, - "auroc": 0.9563335937499999 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4783, - "fn": 17, - "accuracy": 0.9964583333333333 - }, - "0.01": { - "tp": 4710, - "fn": 90, - "accuracy": 0.98125 - } - }, - "auroc": 0.9698013454861111 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4732, - "fn": 68, - "accuracy": 0.9858333333333333 - }, - "0.01": { - "tp": 4652, - "fn": 148, - "accuracy": 0.9691666666666666 - } - }, - "auroc": 0.9704601128472221 - }, - { - "domain": "abstracts", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9515, - "fn": 85, - "accuracy": 0.9911458333333333 - }, - "0.01": { - "tp": 9362, - "fn": 238, - "accuracy": 0.9752083333333333 - } - }, - "auroc": 0.9701307291666668 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2394, - "fn": 6, - "accuracy": 0.9975 - } - }, - "auroc": 0.9933861111111112 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2358, - "fn": 42, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 2330, - "fn": 70, - "accuracy": 0.9708333333333333 - } - }, - "auroc": 0.9654602430555557 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4757, - "fn": 43, - "accuracy": 0.9910416666666667 - }, - "0.01": { - "tp": 4724, - "fn": 76, - "accuracy": 0.9841666666666666 - } - }, - "auroc": 0.9794231770833334 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2248, - "fn": 152, - "accuracy": 0.9366666666666666 - }, - "0.01": { - "tp": 2160, - "fn": 240, - "accuracy": 0.9 - } - }, - "auroc": 0.9207442708333333 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2208, - "fn": 192, - "accuracy": 0.92 - }, - "0.01": { - "tp": 2081, - "fn": 319, - "accuracy": 0.8670833333333333 - } - }, - "auroc": 0.9214902777777777 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4456, - "fn": 344, - "accuracy": 0.9283333333333333 - }, - "0.01": { - "tp": 4241, - "fn": 559, - "accuracy": 0.8835416666666667 - } - }, - "auroc": 0.9211172743055556 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4647, - "fn": 153, - "accuracy": 0.968125 - }, - "0.01": { - "tp": 4554, - "fn": 246, - "accuracy": 0.94875 - } - }, - "auroc": 0.9570651909722223 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4566, - "fn": 234, - "accuracy": 0.95125 - }, - "0.01": { - "tp": 4411, - "fn": 389, - "accuracy": 0.9189583333333333 - } - }, - "auroc": 0.9434752604166667 - }, - { - "domain": "abstracts", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9213, - "fn": 387, - "accuracy": 0.9596875 - }, - "0.01": { - "tp": 8965, - "fn": 635, - "accuracy": 0.9338541666666667 - } - }, - "auroc": 0.9502702256944444 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 2397, - "fn": 3, - "accuracy": 0.99875 - } - }, - "auroc": 0.9949207465277777 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2394, - "fn": 6, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 2375, - "fn": 25, - "accuracy": 0.9895833333333334 - } - }, - "auroc": 0.9893578125 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4792, - "fn": 8, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 4772, - "fn": 28, - "accuracy": 0.9941666666666666 - } - }, - "auroc": 0.9921392795138889 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9914546006944445 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2373, - "fn": 27, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 2352, - "fn": 48, - "accuracy": 0.98 - } - }, - "auroc": 0.9684577256944444 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4771, - "fn": 29, - "accuracy": 0.9939583333333334 - }, - "0.01": { - "tp": 4748, - "fn": 52, - "accuracy": 0.9891666666666666 - } - }, - "auroc": 0.9799561631944445 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4796, - "fn": 4, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 4793, - "fn": 7, - "accuracy": 0.9985416666666667 - } - }, - "auroc": 0.9931876736111112 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4767, - "fn": 33, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 4727, - "fn": 73, - "accuracy": 0.9847916666666666 - } - }, - "auroc": 0.9789077690972222 - }, - { - "domain": "abstracts", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9563, - "fn": 37, - "accuracy": 0.9961458333333333 - }, - "0.01": { - "tp": 9520, - "fn": 80, - "accuracy": 0.9916666666666667 - } - }, - "auroc": 0.9860477213541666 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2231, - "fn": 169, - "accuracy": 0.9295833333333333 - }, - "0.01": { - "tp": 2147, - "fn": 253, - "accuracy": 0.8945833333333333 - } - }, - "auroc": 0.9269026041666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2231, - "fn": 169, - "accuracy": 0.9295833333333333 - }, - "0.01": { - "tp": 2147, - "fn": 253, - "accuracy": 0.8945833333333333 - } - }, - "auroc": 0.9269026041666667 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2238, - "fn": 162, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 2091, - "fn": 309, - "accuracy": 0.87125 - } - }, - "auroc": 0.91761953125 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2238, - "fn": 162, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 2091, - "fn": 309, - "accuracy": 0.87125 - } - }, - "auroc": 0.91761953125 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4469, - "fn": 331, - "accuracy": 0.9310416666666667 - }, - "0.01": { - "tp": 4238, - "fn": 562, - "accuracy": 0.8829166666666667 - } - }, - "auroc": 0.9222610677083334 - }, - { - "domain": "abstracts", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4469, - "fn": 331, - "accuracy": 0.9310416666666667 - }, - "0.01": { - "tp": 4238, - "fn": 562, - "accuracy": 0.8829166666666667 - } - }, - "auroc": 0.9222610677083334 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2011, - "fn": 389, - "accuracy": 0.8379166666666666 - }, - "0.01": { - "tp": 1756, - "fn": 644, - "accuracy": 0.7316666666666667 - } - }, - "auroc": 0.8753013020833332 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2011, - "fn": 389, - "accuracy": 0.8379166666666666 - }, - "0.01": { - "tp": 1756, - "fn": 644, - "accuracy": 0.7316666666666667 - } - }, - "auroc": 0.8753013020833332 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 1893, - "fn": 507, - "accuracy": 0.78875 - }, - "0.01": { - "tp": 1702, - "fn": 698, - "accuracy": 0.7091666666666666 - } - }, - "auroc": 0.8498285590277778 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 1893, - "fn": 507, - "accuracy": 0.78875 - }, - "0.01": { - "tp": 1702, - "fn": 698, - "accuracy": 0.7091666666666666 - } - }, - "auroc": 0.8498285590277778 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 3904, - "fn": 896, - "accuracy": 0.8133333333333334 - }, - "0.01": { - "tp": 3458, - "fn": 1342, - "accuracy": 0.7204166666666667 - } - }, - "auroc": 0.8625649305555556 - }, - { - "domain": "abstracts", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 3904, - "fn": 896, - "accuracy": 0.8133333333333334 - }, - "0.01": { - "tp": 3458, - "fn": 1342, - "accuracy": 0.7204166666666667 - } - }, - "auroc": 0.8625649305555556 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9942858506944444 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9942858506944444 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9924986979166667 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9924986979166667 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - }, - "0.01": { - "tp": 4792, - "fn": 8, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9933922743055555 - }, - { - "domain": "abstracts", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - }, - "0.01": { - "tp": 4792, - "fn": 8, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9933922743055555 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - } - }, - "auroc": 0.9930466145833333 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - } - }, - "auroc": 0.9930466145833333 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2393, - "fn": 7, - "accuracy": 0.9970833333333333 - }, - "0.01": { - "tp": 2375, - "fn": 25, - "accuracy": 0.9895833333333334 - } - }, - "auroc": 0.9720162326388888 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2393, - "fn": 7, - "accuracy": 0.9970833333333333 - }, - "0.01": { - "tp": 2375, - "fn": 25, - "accuracy": 0.9895833333333334 - } - }, - "auroc": 0.9720162326388888 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4792, - "fn": 8, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 4773, - "fn": 27, - "accuracy": 0.994375 - } - }, - "auroc": 0.982531423611111 - }, - { - "domain": "abstracts", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4792, - "fn": 8, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 4773, - "fn": 27, - "accuracy": 0.994375 - } - }, - "auroc": 0.982531423611111 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2323, - "fn": 77, - "accuracy": 0.9679166666666666 - }, - "0.01": { - "tp": 2266, - "fn": 134, - "accuracy": 0.9441666666666667 - } - }, - "auroc": 0.9635276909722222 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2323, - "fn": 77, - "accuracy": 0.9679166666666666 - }, - "0.01": { - "tp": 2266, - "fn": 134, - "accuracy": 0.9441666666666667 - } - }, - "auroc": 0.9635276909722222 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2261, - "fn": 139, - "accuracy": 0.9420833333333334 - }, - "0.01": { - "tp": 2157, - "fn": 243, - "accuracy": 0.89875 - } - }, - "auroc": 0.9465157118055556 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2261, - "fn": 139, - "accuracy": 0.9420833333333334 - }, - "0.01": { - "tp": 2157, - "fn": 243, - "accuracy": 0.89875 - } - }, - "auroc": 0.9465157118055556 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4584, - "fn": 216, - "accuracy": 0.955 - }, - "0.01": { - "tp": 4423, - "fn": 377, - "accuracy": 0.9214583333333334 - } - }, - "auroc": 0.9550217013888889 - }, - { - "domain": "abstracts", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4584, - "fn": 216, - "accuracy": 0.955 - }, - "0.01": { - "tp": 4423, - "fn": 377, - "accuracy": 0.9214583333333334 - } - }, - "auroc": 0.9550217013888889 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 25744, - "fn": 656, - "accuracy": 0.9751515151515151 - }, - "0.01": { - "tp": 25295, - "fn": 1105, - "accuracy": 0.9581439393939394 - } - }, - "auroc": 0.9714218276515152 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 14290, - "fn": 110, - "accuracy": 0.9923611111111111 - }, - "0.01": { - "tp": 14189, - "fn": 211, - "accuracy": 0.9853472222222223 - } - }, - "auroc": 0.9834519386574073 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 40034, - "fn": 766, - "accuracy": 0.9812254901960784 - }, - "0.01": { - "tp": 39484, - "fn": 1316, - "accuracy": 0.9677450980392157 - } - }, - "auroc": 0.9756677491830065 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 25261, - "fn": 1139, - "accuracy": 0.9568560606060607 - }, - "0.01": { - "tp": 24602, - "fn": 1798, - "accuracy": 0.9318939393939394 - } - }, - "auroc": 0.9517261284722223 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 14029, - "fn": 371, - "accuracy": 0.9742361111111111 - }, - "0.01": { - "tp": 13682, - "fn": 718, - "accuracy": 0.9501388888888889 - } - }, - "auroc": 0.9611311921296295 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 39290, - "fn": 1510, - "accuracy": 0.9629901960784314 - }, - "0.01": { - "tp": 38284, - "fn": 2516, - "accuracy": 0.9383333333333334 - } - }, - "auroc": 0.9550455627042485 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 51005, - "fn": 1795, - "accuracy": 0.9660037878787879 - }, - "0.01": { - "tp": 49897, - "fn": 2903, - "accuracy": 0.9450189393939394 - } - }, - "auroc": 0.9615739780618687 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 28319, - "fn": 481, - "accuracy": 0.9832986111111112 - }, - "0.01": { - "tp": 27871, - "fn": 929, - "accuracy": 0.9677430555555555 - } - }, - "auroc": 0.9722915653935185 - }, - { - "domain": "abstracts", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 79324, - "fn": 2276, - "accuracy": 0.972107843137255 - }, - "0.01": { - "tp": 77768, - "fn": 3832, - "accuracy": 0.9530392156862745 - } - }, - "auroc": 0.9653566559436275 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999708333333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999854166666666 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9951895833333333 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9975947916666666 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9975802083333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9987901041666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9943510416666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974791666666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9959151041666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - }, - "0.01": { - "tp": 146, - "fn": 54, - "accuracy": 0.73 - } - }, - "auroc": 0.9041854166666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9798604166666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - }, - "0.01": { - "tp": 338, - "fn": 62, - "accuracy": 0.845 - } - }, - "auroc": 0.9420229166666668 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 343, - "fn": 57, - "accuracy": 0.8575 - } - }, - "auroc": 0.9492682291666668 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9886697916666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - }, - "0.01": { - "tp": 734, - "fn": 66, - "accuracy": 0.9175 - } - }, - "auroc": 0.9689690104166667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.992946875 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9964718749999999 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999729166666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9792645833333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.98961875 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999848958333334 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9861057291666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9930453124999999 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9763072916666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992614583333334 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9877843749999999 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9487760416666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9568343750000001 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 359, - "fn": 41, - "accuracy": 0.8975 - } - }, - "auroc": 0.9528052083333333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9625416666666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9780479166666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 778, - "fn": 22, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 755, - "fn": 45, - "accuracy": 0.94375 - } - }, - "auroc": 0.9702947916666667 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9988729166666667 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.997546875 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9982098958333334 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - } - }, - "auroc": 0.9126593749999999 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.957565625 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 326, - "fn": 74, - "accuracy": 0.815 - } - }, - "auroc": 0.9351125 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - } - }, - "auroc": 0.9557661458333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.97755625 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 726, - "fn": 74, - "accuracy": 0.9075 - } - }, - "auroc": 0.9666611979166667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999802083333332 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999890625 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9972989583333334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986484375 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986395833333334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99931875 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.988271875 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.988271875 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9779406250000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9779406250000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9831062500000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9831062500000001 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.966665625 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.966665625 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9258541666666666 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9258541666666666 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9462598958333335 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9462598958333335 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999984375000001 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999984375000001 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9900260416666666 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9900260416666666 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9950130208333333 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9950130208333333 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9844625 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9844625 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9693427083333335 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9693427083333335 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9769026041666667 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9769026041666667 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2188, - "fn": 12, - "accuracy": 0.9945454545454545 - }, - "0.01": { - "tp": 2167, - "fn": 33, - "accuracy": 0.985 - } - }, - "auroc": 0.9917205492424243 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9978642361111111 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3386, - "fn": 14, - "accuracy": 0.9958823529411764 - }, - "0.01": { - "tp": 3365, - "fn": 35, - "accuracy": 0.9897058823529412 - } - }, - "auroc": 0.9938889093137255 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2107, - "fn": 93, - "accuracy": 0.9577272727272728 - }, - "0.01": { - "tp": 2009, - "fn": 191, - "accuracy": 0.9131818181818182 - } - }, - "auroc": 0.9662501893939395 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1177, - "fn": 23, - "accuracy": 0.9808333333333333 - }, - "0.01": { - "tp": 1143, - "fn": 57, - "accuracy": 0.9525 - } - }, - "auroc": 0.9776689236111111 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3284, - "fn": 116, - "accuracy": 0.9658823529411765 - }, - "0.01": { - "tp": 3152, - "fn": 248, - "accuracy": 0.9270588235294117 - } - }, - "auroc": 0.970280330882353 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 4295, - "fn": 105, - "accuracy": 0.9761363636363637 - }, - "0.01": { - "tp": 4176, - "fn": 224, - "accuracy": 0.9490909090909091 - } - }, - "auroc": 0.9789853693181818 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2375, - "fn": 25, - "accuracy": 0.9895833333333334 - }, - "0.01": { - "tp": 2341, - "fn": 59, - "accuracy": 0.9754166666666667 - } - }, - "auroc": 0.987766579861111 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 6670, - "fn": 130, - "accuracy": 0.9808823529411764 - }, - "0.01": { - "tp": 6517, - "fn": 283, - "accuracy": 0.9583823529411765 - } - }, - "auroc": 0.9820846200980393 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999708333333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999854166666666 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9951895833333333 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9975947916666666 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9975802083333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9987901041666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9943510416666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974791666666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9959151041666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - }, - "0.01": { - "tp": 146, - "fn": 54, - "accuracy": 0.73 - } - }, - "auroc": 0.9041854166666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9798604166666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - }, - "0.01": { - "tp": 338, - "fn": 62, - "accuracy": 0.845 - } - }, - "auroc": 0.9420229166666668 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 343, - "fn": 57, - "accuracy": 0.8575 - } - }, - "auroc": 0.9492682291666668 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9886697916666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - }, - "0.01": { - "tp": 734, - "fn": 66, - "accuracy": 0.9175 - } - }, - "auroc": 0.9689690104166667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.992946875 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9964718749999999 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999729166666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9792645833333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.98961875 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999848958333334 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9861057291666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9930453124999999 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9763072916666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992614583333334 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9877843749999999 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9487760416666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9568343750000001 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 359, - "fn": 41, - "accuracy": 0.8975 - } - }, - "auroc": 0.9528052083333333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9625416666666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9780479166666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 778, - "fn": 22, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 755, - "fn": 45, - "accuracy": 0.94375 - } - }, - "auroc": 0.9702947916666667 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9988729166666667 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.997546875 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9982098958333334 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - } - }, - "auroc": 0.9126593749999999 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.957565625 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 326, - "fn": 74, - "accuracy": 0.815 - } - }, - "auroc": 0.9351125 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - } - }, - "auroc": 0.9557661458333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.97755625 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 726, - "fn": 74, - "accuracy": 0.9075 - } - }, - "auroc": 0.9666611979166667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999802083333332 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999890625 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9972989583333334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986484375 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986395833333334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99931875 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.988271875 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.988271875 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9779406250000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9779406250000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9831062500000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9831062500000001 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.966665625 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.966665625 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9258541666666666 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9258541666666666 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9462598958333335 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9462598958333335 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999984375000001 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999984375000001 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9900260416666666 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9900260416666666 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9950130208333333 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9950130208333333 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9844625 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9844625 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9693427083333335 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9693427083333335 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9769026041666667 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9769026041666667 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2188, - "fn": 12, - "accuracy": 0.9945454545454545 - }, - "0.01": { - "tp": 2167, - "fn": 33, - "accuracy": 0.985 - } - }, - "auroc": 0.9917205492424243 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9978642361111111 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3386, - "fn": 14, - "accuracy": 0.9958823529411764 - }, - "0.01": { - "tp": 3365, - "fn": 35, - "accuracy": 0.9897058823529412 - } - }, - "auroc": 0.9938889093137255 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2107, - "fn": 93, - "accuracy": 0.9577272727272728 - }, - "0.01": { - "tp": 2009, - "fn": 191, - "accuracy": 0.9131818181818182 - } - }, - "auroc": 0.9662501893939395 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1177, - "fn": 23, - "accuracy": 0.9808333333333333 - }, - "0.01": { - "tp": 1143, - "fn": 57, - "accuracy": 0.9525 - } - }, - "auroc": 0.9776689236111111 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3284, - "fn": 116, - "accuracy": 0.9658823529411765 - }, - "0.01": { - "tp": 3152, - "fn": 248, - "accuracy": 0.9270588235294117 - } - }, - "auroc": 0.970280330882353 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 4295, - "fn": 105, - "accuracy": 0.9761363636363637 - }, - "0.01": { - "tp": 4176, - "fn": 224, - "accuracy": 0.9490909090909091 - } - }, - "auroc": 0.9789853693181818 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2375, - "fn": 25, - "accuracy": 0.9895833333333334 - }, - "0.01": { - "tp": 2341, - "fn": 59, - "accuracy": 0.9754166666666667 - } - }, - "auroc": 0.987766579861111 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 6670, - "fn": 130, - "accuracy": 0.9808823529411764 - }, - "0.01": { - "tp": 6517, - "fn": 283, - "accuracy": 0.9583823529411765 - } - }, - "auroc": 0.9820846200980393 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999791666666668 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997135416666667 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998463541666667 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99985625 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9965677083333333 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9982119791666666 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999177083333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998140625 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990291666666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998440625 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9964343750000001 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9974375 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9711239583333333 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9767760416666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.97395 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9847822916666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9866052083333334 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 784, - "fn": 16, - "accuracy": 0.98 - } - }, - "auroc": 0.98569375 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992770833333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9934802083333334 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9963786458333335 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992770833333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9778364583333332 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9885567708333334 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992770833333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9856583333333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9924677083333333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9961322916666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997020833333333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9979171874999999 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9819989583333333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9816947916666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.981846875 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.989065625 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9906984375000001 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 784, - "fn": 16, - "accuracy": 0.98 - } - }, - "auroc": 0.9898820312500001 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998864583333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993197916666666 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996031249999999 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9745000000000001 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.973075 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9737875 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9871932291666666 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9861973958333334 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 788, - "fn": 12, - "accuracy": 0.985 - } - }, - "auroc": 0.9866953125 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999791666666666 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999614583333333 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999703125 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997406249999999 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9971687499999999 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9984546875 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998598958333333 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985651041666666 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992125 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.98866875 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.98866875 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9811437500000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9811437500000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.98490625 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.98490625 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.983953125 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.983953125 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.956834375 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.956834375 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9703937499999999 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9703937499999999 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994677083333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994677083333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980427083333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980427083333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987552083333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987552083333333 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994135416666666 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994135416666666 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9807479166666666 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9807479166666666 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9900807291666667 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9900807291666667 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.99075 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.99075 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9819458333333334 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9819458333333334 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9863479166666667 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9863479166666667 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2198, - "fn": 2, - "accuracy": 0.9990909090909091 - }, - "0.01": { - "tp": 2188, - "fn": 12, - "accuracy": 0.9945454545454545 - } - }, - "auroc": 0.9959952651515152 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9981019097222222 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3396, - "fn": 4, - "accuracy": 0.9988235294117647 - }, - "0.01": { - "tp": 3386, - "fn": 14, - "accuracy": 0.9958823529411764 - } - }, - "auroc": 0.9967387867647058 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2189, - "fn": 11, - "accuracy": 0.995 - }, - "0.01": { - "tp": 2156, - "fn": 44, - "accuracy": 0.98 - } - }, - "auroc": 0.9841101325757576 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1190, - "fn": 10, - "accuracy": 0.9916666666666667 - }, - "0.01": { - "tp": 1173, - "fn": 27, - "accuracy": 0.9775 - } - }, - "auroc": 0.9838531249999999 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3379, - "fn": 21, - "accuracy": 0.9938235294117647 - }, - "0.01": { - "tp": 3329, - "fn": 71, - "accuracy": 0.9791176470588235 - } - }, - "auroc": 0.9840194240196078 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 4387, - "fn": 13, - "accuracy": 0.9970454545454546 - }, - "0.01": { - "tp": 4344, - "fn": 56, - "accuracy": 0.9872727272727273 - } - }, - "auroc": 0.9900526988636362 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2388, - "fn": 12, - "accuracy": 0.995 - }, - "0.01": { - "tp": 2371, - "fn": 29, - "accuracy": 0.9879166666666667 - } - }, - "auroc": 0.9909775173611112 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 6775, - "fn": 25, - "accuracy": 0.9963235294117647 - }, - "0.01": { - "tp": 6715, - "fn": 85, - "accuracy": 0.9875 - } - }, - "auroc": 0.9903791053921569 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999916666666667 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989145833333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999453125 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999903125 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.985553125 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.992728125 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999473958333333 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9922338541666667 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9960906250000001 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9954572916666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.99668125 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9960692708333333 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - }, - "0.01": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - } - }, - "auroc": 0.8963833333333333 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9777322916666668 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 327, - "fn": 73, - "accuracy": 0.8175 - } - }, - "auroc": 0.9370578125 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - }, - "0.01": { - "tp": 333, - "fn": 67, - "accuracy": 0.8325 - } - }, - "auroc": 0.9459203125000001 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9872067708333333 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 763, - "fn": 37, - "accuracy": 0.95375 - }, - "0.01": { - "tp": 725, - "fn": 75, - "accuracy": 0.90625 - } - }, - "auroc": 0.9665635416666666 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.997371875 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9857770833333334 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9915744791666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99585625 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9700374999999999 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9829468750000001 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9966140625000001 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9779072916666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - } - }, - "auroc": 0.9872606770833333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9760239583333334 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9962906250000001 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9861572916666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9332875 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9576708333333335 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 347, - "fn": 53, - "accuracy": 0.8675 - } - }, - "auroc": 0.9454791666666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - } - }, - "auroc": 0.9546557291666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9769807291666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 777, - "fn": 23, - "accuracy": 0.97125 - }, - "0.01": { - "tp": 742, - "fn": 58, - "accuracy": 0.9275 - } - }, - "auroc": 0.9658182291666666 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992135416666666 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9951270833333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9971703125 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - }, - "0.01": { - "tp": 136, - "fn": 64, - "accuracy": 0.68 - } - }, - "auroc": 0.8996375 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9581083333333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - } - }, - "auroc": 0.9288729166666667 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 336, - "fn": 64, - "accuracy": 0.84 - } - }, - "auroc": 0.9494255208333334 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9766177083333334 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 775, - "fn": 25, - "accuracy": 0.96875 - }, - "0.01": { - "tp": 718, - "fn": 82, - "accuracy": 0.8975 - } - }, - "auroc": 0.9630216145833334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999343749999999 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999225 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995796875 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990177083333334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9863197916666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.99266875 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994760416666666 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9927723958333332 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - } - }, - "auroc": 0.9961242187500001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9703395833333333 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9703395833333333 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9550937500000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9550937500000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.9627166666666667 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.9627166666666667 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 160, - "fn": 40, - "accuracy": 0.8 - } - }, - "auroc": 0.9383239583333334 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 160, - "fn": 40, - "accuracy": 0.8 - } - }, - "auroc": 0.9383239583333334 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.9058625 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.9058625 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - }, - "0.01": { - "tp": 307, - "fn": 93, - "accuracy": 0.7675 - } - }, - "auroc": 0.9220932291666667 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - }, - "0.01": { - "tp": 307, - "fn": 93, - "accuracy": 0.7675 - } - }, - "auroc": 0.9220932291666667 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999625 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999625 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996083333333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996083333333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997854166666666 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997854166666666 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999750000000001 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999750000000001 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9821916666666667 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9821916666666667 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9910833333333333 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9910833333333333 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9643572916666666 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9643572916666666 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.946296875 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.946296875 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 348, - "fn": 52, - "accuracy": 0.87 - } - }, - "auroc": 0.9553270833333334 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 348, - "fn": 52, - "accuracy": 0.87 - } - }, - "auroc": 0.9553270833333334 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2177, - "fn": 23, - "accuracy": 0.9895454545454545 - }, - "0.01": { - "tp": 2118, - "fn": 82, - "accuracy": 0.9627272727272728 - } - }, - "auroc": 0.9855410037878788 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1196, - "fn": 4, - "accuracy": 0.9966666666666667 - } - }, - "auroc": 0.9953359375 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3375, - "fn": 25, - "accuracy": 0.9926470588235294 - }, - "0.01": { - "tp": 3314, - "fn": 86, - "accuracy": 0.9747058823529412 - } - }, - "auroc": 0.9889980392156863 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2086, - "fn": 114, - "accuracy": 0.9481818181818182 - }, - "0.01": { - "tp": 1923, - "fn": 277, - "accuracy": 0.8740909090909091 - } - }, - "auroc": 0.9557398674242423 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1176, - "fn": 24, - "accuracy": 0.98 - }, - "0.01": { - "tp": 1136, - "fn": 64, - "accuracy": 0.9466666666666667 - } - }, - "auroc": 0.9725703125 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3262, - "fn": 138, - "accuracy": 0.9594117647058824 - }, - "0.01": { - "tp": 3059, - "fn": 341, - "accuracy": 0.8997058823529411 - } - }, - "auroc": 0.9616800245098038 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 4263, - "fn": 137, - "accuracy": 0.9688636363636364 - }, - "0.01": { - "tp": 4041, - "fn": 359, - "accuracy": 0.918409090909091 - } - }, - "auroc": 0.9706404356060605 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2374, - "fn": 26, - "accuracy": 0.9891666666666666 - }, - "0.01": { - "tp": 2332, - "fn": 68, - "accuracy": 0.9716666666666667 - } - }, - "auroc": 0.983953125 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 6637, - "fn": 163, - "accuracy": 0.9760294117647059 - }, - "0.01": { - "tp": 6373, - "fn": 427, - "accuracy": 0.9372058823529412 - } - }, - "auroc": 0.9753390318627451 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9964791666666667 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9982395833333333 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9982395833333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9991197916666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9944541666666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9976114583333333 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9960328125 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.9411989583333333 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9791697916666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - } - }, - "auroc": 0.960184375 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9678265625 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9883906250000001 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 783, - "fn": 17, - "accuracy": 0.97875 - }, - "0.01": { - "tp": 760, - "fn": 40, - "accuracy": 0.95 - } - }, - "auroc": 0.97810859375 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9934583333333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9967291666666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999729166666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9798927083333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9899328125 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999864583333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9866755208333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9933309895833333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9762010416666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99951875 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9878598958333333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9657114583333333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9702239583333334 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9679677083333333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.97095625 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9848713541666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 782, - "fn": 18, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 765, - "fn": 35, - "accuracy": 0.95625 - } - }, - "auroc": 0.9779138020833333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998965625 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989625 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989640625 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9507406249999999 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9626125000000001 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9566765625 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9748531250000001 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9807875 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 786, - "fn": 14, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 756, - "fn": 44, - "accuracy": 0.945 - } - }, - "auroc": 0.9778203125 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99999375 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.99854375 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.999271875 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9992687499999999 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9996343750000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.987759375 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.987759375 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9786062500000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9786062500000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9831828125000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9831828125000001 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9764343750000001 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9764343750000001 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.940015625 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.940015625 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.958225 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.958225 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999833333333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999833333333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999916666666667 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999916666666667 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9926291666666667 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9926291666666667 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9963145833333333 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9963145833333333 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.987159375 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.987159375 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.976890625 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.976890625 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.982025 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.982025 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2189, - "fn": 11, - "accuracy": 0.995 - }, - "0.01": { - "tp": 2174, - "fn": 26, - "accuracy": 0.9881818181818182 - } - }, - "auroc": 0.9928158143939394 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9982574652777778 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3387, - "fn": 13, - "accuracy": 0.9961764705882353 - }, - "0.01": { - "tp": 3372, - "fn": 28, - "accuracy": 0.991764705882353 - } - }, - "auroc": 0.9947363970588236 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2146, - "fn": 54, - "accuracy": 0.9754545454545455 - }, - "0.01": { - "tp": 2083, - "fn": 117, - "accuracy": 0.9468181818181818 - } - }, - "auroc": 0.9768862689393939 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1180, - "fn": 20, - "accuracy": 0.9833333333333333 - }, - "0.01": { - "tp": 1152, - "fn": 48, - "accuracy": 0.96 - } - }, - "auroc": 0.9811536458333334 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3326, - "fn": 74, - "accuracy": 0.9782352941176471 - }, - "0.01": { - "tp": 3235, - "fn": 165, - "accuracy": 0.9514705882352941 - } - }, - "auroc": 0.9783924019607843 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 4335, - "fn": 65, - "accuracy": 0.9852272727272727 - }, - "0.01": { - "tp": 4257, - "fn": 143, - "accuracy": 0.9675 - } - }, - "auroc": 0.9848510416666667 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2378, - "fn": 22, - "accuracy": 0.9908333333333333 - }, - "0.01": { - "tp": 2350, - "fn": 50, - "accuracy": 0.9791666666666666 - } - }, - "auroc": 0.9897055555555556 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 6713, - "fn": 87, - "accuracy": 0.9872058823529412 - }, - "0.01": { - "tp": 6607, - "fn": 193, - "accuracy": 0.9716176470588235 - } - }, - "auroc": 0.9865643995098039 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987197916666667 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9918114583333333 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9952656249999998 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99661875 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9775687500000001 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9870937500000001 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9976692708333332 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9846901041666667 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 787, - "fn": 13, - "accuracy": 0.98375 - } - }, - "auroc": 0.9911796875 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9924875 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - }, - "0.01": { - "tp": 126, - "fn": 74, - "accuracy": 0.63 - } - }, - "auroc": 0.88539375 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 325, - "fn": 75, - "accuracy": 0.8125 - } - }, - "auroc": 0.938940625 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.945634375 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 150, - "fn": 50, - "accuracy": 0.75 - }, - "0.01": { - "tp": 97, - "fn": 103, - "accuracy": 0.485 - } - }, - "auroc": 0.8390979166666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 341, - "fn": 59, - "accuracy": 0.8525 - }, - "0.01": { - "tp": 269, - "fn": 131, - "accuracy": 0.6725 - } - }, - "auroc": 0.8923661458333334 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9690609375 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - }, - "0.01": { - "tp": 223, - "fn": 177, - "accuracy": 0.5575 - } - }, - "auroc": 0.8622458333333333 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 723, - "fn": 77, - "accuracy": 0.90375 - }, - "0.01": { - "tp": 594, - "fn": 206, - "accuracy": 0.7425 - } - }, - "auroc": 0.9156533854166667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9774385416666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.9379385416666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9576885416666666 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9770395833333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - } - }, - "auroc": 0.9122520833333334 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - } - }, - "auroc": 0.9446458333333334 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9772390625 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 312, - "fn": 88, - "accuracy": 0.78 - } - }, - "auroc": 0.9250953124999999 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 772, - "fn": 28, - "accuracy": 0.965 - }, - "0.01": { - "tp": 692, - "fn": 108, - "accuracy": 0.865 - } - }, - "auroc": 0.9511671875000001 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.972046875 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9844489583333333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9782479166666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9504520833333334 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - }, - "0.01": { - "tp": 114, - "fn": 86, - "accuracy": 0.57 - } - }, - "auroc": 0.870171875 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - }, - "0.01": { - "tp": 286, - "fn": 114, - "accuracy": 0.715 - } - }, - "auroc": 0.9103119791666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9612494791666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - }, - "0.01": { - "tp": 311, - "fn": 89, - "accuracy": 0.7775 - } - }, - "auroc": 0.9273104166666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 748, - "fn": 52, - "accuracy": 0.935 - }, - "0.01": { - "tp": 676, - "fn": 124, - "accuracy": 0.845 - } - }, - "auroc": 0.9442799479166667 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99631875 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9548 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.975559375 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9382177083333334 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 131, - "fn": 69, - "accuracy": 0.655 - }, - "0.01": { - "tp": 77, - "fn": 123, - "accuracy": 0.385 - } - }, - "auroc": 0.8120833333333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 320, - "fn": 80, - "accuracy": 0.8 - }, - "0.01": { - "tp": 241, - "fn": 159, - "accuracy": 0.6025 - } - }, - "auroc": 0.8751505208333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - } - }, - "auroc": 0.9672682291666667 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 325, - "fn": 75, - "accuracy": 0.8125 - }, - "0.01": { - "tp": 253, - "fn": 147, - "accuracy": 0.6325 - } - }, - "auroc": 0.8834416666666667 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 714, - "fn": 86, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 617, - "fn": 183, - "accuracy": 0.77125 - } - }, - "auroc": 0.9253549479166667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9941802083333333 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.990659375 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9924197916666666 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9914291666666666 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9667239583333334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9790765625 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9928046875000001 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9786916666666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 778, - "fn": 22, - "accuracy": 0.9725 - } - }, - "auroc": 0.9857481770833333 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - } - }, - "auroc": 0.9463541666666667 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - } - }, - "auroc": 0.9463541666666667 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.94183125 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.94183125 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - }, - "0.01": { - "tp": 331, - "fn": 69, - "accuracy": 0.8275 - } - }, - "auroc": 0.9440927083333333 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - }, - "0.01": { - "tp": 331, - "fn": 69, - "accuracy": 0.8275 - } - }, - "auroc": 0.9440927083333333 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.9509083333333334 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.9509083333333334 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 153, - "fn": 47, - "accuracy": 0.765 - } - }, - "auroc": 0.9228895833333334 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 153, - "fn": 47, - "accuracy": 0.765 - } - }, - "auroc": 0.9228895833333334 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 327, - "fn": 73, - "accuracy": 0.8175 - } - }, - "auroc": 0.9368989583333333 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 327, - "fn": 73, - "accuracy": 0.8175 - } - }, - "auroc": 0.9368989583333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9952395833333334 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9952395833333334 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9905416666666667 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9905416666666667 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.992890625 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.992890625 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9947333333333335 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9947333333333335 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9785989583333333 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9785989583333333 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9866661458333333 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9866661458333333 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9651093749999999 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9651093749999999 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9485260416666668 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9485260416666668 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - } - }, - "auroc": 0.9568177083333333 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - } - }, - "auroc": 0.9568177083333333 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2164, - "fn": 36, - "accuracy": 0.9836363636363636 - }, - "0.01": { - "tp": 2097, - "fn": 103, - "accuracy": 0.9531818181818181 - } - }, - "auroc": 0.9803214962121212 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1167, - "fn": 33, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 1055, - "fn": 145, - "accuracy": 0.8791666666666667 - } - }, - "auroc": 0.9575086805555556 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3331, - "fn": 69, - "accuracy": 0.9797058823529412 - }, - "0.01": { - "tp": 3152, - "fn": 248, - "accuracy": 0.9270588235294117 - } - }, - "auroc": 0.9722699142156862 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2116, - "fn": 84, - "accuracy": 0.9618181818181818 - }, - "0.01": { - "tp": 1977, - "fn": 223, - "accuracy": 0.8986363636363637 - } - }, - "auroc": 0.9619799242424243 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1020, - "fn": 180, - "accuracy": 0.85 - }, - "0.01": { - "tp": 812, - "fn": 388, - "accuracy": 0.6766666666666666 - } - }, - "auroc": 0.8963163194444446 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3136, - "fn": 264, - "accuracy": 0.9223529411764706 - }, - "0.01": { - "tp": 2789, - "fn": 611, - "accuracy": 0.8202941176470588 - } - }, - "auroc": 0.9388045343137255 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 4280, - "fn": 120, - "accuracy": 0.9727272727272728 - }, - "0.01": { - "tp": 4074, - "fn": 326, - "accuracy": 0.9259090909090909 - } - }, - "auroc": 0.9711507102272727 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2187, - "fn": 213, - "accuracy": 0.91125 - }, - "0.01": { - "tp": 1867, - "fn": 533, - "accuracy": 0.7779166666666667 - } - }, - "auroc": 0.9269125 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 6467, - "fn": 333, - "accuracy": 0.9510294117647059 - }, - "0.01": { - "tp": 5941, - "fn": 859, - "accuracy": 0.8736764705882353 - } - }, - "auroc": 0.955537224264706 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999708333333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999854166666666 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9951895833333333 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9975947916666666 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9975802083333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9987901041666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9957166666666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9975093749999999 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9966130208333333 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 154, - "fn": 46, - "accuracy": 0.77 - } - }, - "auroc": 0.9207635416666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9801260416666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 346, - "fn": 54, - "accuracy": 0.865 - } - }, - "auroc": 0.9504447916666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.9582401041666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9888177083333333 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 774, - "fn": 26, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 744, - "fn": 56, - "accuracy": 0.93 - } - }, - "auroc": 0.97352890625 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9929635416666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9964802083333334 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999729166666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9792 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9895864583333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999848958333334 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9860817708333334 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9930333333333334 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9793749999999999 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995239583333333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9894494791666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.958921875 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.963928125 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.961425 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9691484375 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9817260416666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 780, - "fn": 20, - "accuracy": 0.975 - }, - "0.01": { - "tp": 762, - "fn": 38, - "accuracy": 0.9525 - } - }, - "auroc": 0.9754372395833334 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99913125 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99786875 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9984999999999999 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - }, - "0.01": { - "tp": 157, - "fn": 43, - "accuracy": 0.785 - } - }, - "auroc": 0.9230572916666666 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9601708333333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 335, - "fn": 65, - "accuracy": 0.8375 - } - }, - "auroc": 0.9416140625 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9610942708333334 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9790197916666666 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 776, - "fn": 24, - "accuracy": 0.97 - }, - "0.01": { - "tp": 735, - "fn": 65, - "accuracy": 0.91875 - } - }, - "auroc": 0.97005703125 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999802083333333 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999890625 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9973041666666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986510416666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986421875 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993200520833333 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9880802083333332 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9880802083333332 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9776968750000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9776968750000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9828885416666668 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9828885416666668 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.968984375 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.968984375 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.9323645833333334 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.9323645833333334 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.9506744791666667 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.9506744791666667 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999984375000001 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999984375000001 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.990059375 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.990059375 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9950296874999999 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9950296874999999 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9844812500000001 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9844812500000001 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9711520833333334 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9711520833333334 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9778166666666667 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9778166666666667 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2187, - "fn": 13, - "accuracy": 0.9940909090909091 - }, - "0.01": { - "tp": 2169, - "fn": 31, - "accuracy": 0.985909090909091 - } - }, - "auroc": 0.9923421401515151 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9979694444444445 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3385, - "fn": 15, - "accuracy": 0.9955882352941177 - }, - "0.01": { - "tp": 3367, - "fn": 33, - "accuracy": 0.9902941176470588 - } - }, - "auroc": 0.9943282475490196 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2119, - "fn": 81, - "accuracy": 0.9631818181818181 - }, - "0.01": { - "tp": 2034, - "fn": 166, - "accuracy": 0.9245454545454546 - } - }, - "auroc": 0.9703621212121212 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1183, - "fn": 17, - "accuracy": 0.9858333333333333 - }, - "0.01": { - "tp": 1148, - "fn": 52, - "accuracy": 0.9566666666666667 - } - }, - "auroc": 0.9793197916666667 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3302, - "fn": 98, - "accuracy": 0.9711764705882353 - }, - "0.01": { - "tp": 3182, - "fn": 218, - "accuracy": 0.9358823529411765 - } - }, - "auroc": 0.9735236519607843 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 4306, - "fn": 94, - "accuracy": 0.9786363636363636 - }, - "0.01": { - "tp": 4203, - "fn": 197, - "accuracy": 0.9552272727272727 - } - }, - "auroc": 0.9813521306818183 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2381, - "fn": 19, - "accuracy": 0.9920833333333333 - }, - "0.01": { - "tp": 2346, - "fn": 54, - "accuracy": 0.9775 - } - }, - "auroc": 0.9886446180555555 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 6687, - "fn": 113, - "accuracy": 0.9833823529411765 - }, - "0.01": { - "tp": 6549, - "fn": 251, - "accuracy": 0.9630882352941177 - } - }, - "auroc": 0.983925949754902 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999708333333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999854166666666 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9952604166666666 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9976302083333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9976156250000001 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9988078125000001 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9943552083333334 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974395833333334 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9958973958333334 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - }, - "0.01": { - "tp": 145, - "fn": 55, - "accuracy": 0.725 - } - }, - "auroc": 0.9034302083333333 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9792947916666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - }, - "0.01": { - "tp": 337, - "fn": 63, - "accuracy": 0.8425 - } - }, - "auroc": 0.9413625000000001 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - } - }, - "auroc": 0.9488927083333334 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9883671875 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - }, - "0.01": { - "tp": 733, - "fn": 67, - "accuracy": 0.91625 - } - }, - "auroc": 0.9686299479166668 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9931218749999999 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9965598958333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999975 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9801739583333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9900744791666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999864583333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9866479166666666 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9933171875 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9765229166666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993135416666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9879182291666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9483697916666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9563604166666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9523651041666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9624463541666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9778369791666668 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 778, - "fn": 22, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 754, - "fn": 46, - "accuracy": 0.9425 - } - }, - "auroc": 0.9701416666666667 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989458333333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9975020833333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9982239583333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - } - }, - "auroc": 0.9111416666666667 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9564906249999999 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - }, - "0.01": { - "tp": 326, - "fn": 74, - "accuracy": 0.815 - } - }, - "auroc": 0.9338161458333334 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - } - }, - "auroc": 0.95504375 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9769963541666667 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 767, - "fn": 33, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 726, - "fn": 74, - "accuracy": 0.9075 - } - }, - "auroc": 0.9660200520833334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99998125 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999990625 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9972916666666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986447916666666 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999989583333333 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986364583333334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993177083333333 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9884489583333333 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9884489583333333 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9788895833333333 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9788895833333333 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9836692708333334 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9836692708333334 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9663875 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9663875 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9266614583333334 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9266614583333334 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9465244791666667 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9465244791666667 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99999375 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99999375 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9900979166666667 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9900979166666667 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9950489583333333 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9950489583333333 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9842822916666667 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9842822916666667 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9692343750000001 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9692343750000001 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9767583333333334 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9767583333333334 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2188, - "fn": 12, - "accuracy": 0.9945454545454545 - }, - "0.01": { - "tp": 2167, - "fn": 33, - "accuracy": 0.985 - } - }, - "auroc": 0.9917218750000001 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9978881944444444 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3386, - "fn": 14, - "accuracy": 0.9958823529411764 - }, - "0.01": { - "tp": 3365, - "fn": 35, - "accuracy": 0.9897058823529412 - } - }, - "auroc": 0.9938982230392157 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2106, - "fn": 94, - "accuracy": 0.9572727272727273 - }, - "0.01": { - "tp": 2007, - "fn": 193, - "accuracy": 0.9122727272727272 - } - }, - "auroc": 0.9661628787878789 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1175, - "fn": 25, - "accuracy": 0.9791666666666666 - }, - "0.01": { - "tp": 1142, - "fn": 58, - "accuracy": 0.9516666666666667 - } - }, - "auroc": 0.9774786458333333 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3281, - "fn": 119, - "accuracy": 0.965 - }, - "0.01": { - "tp": 3149, - "fn": 251, - "accuracy": 0.9261764705882353 - } - }, - "auroc": 0.9701566789215685 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 4294, - "fn": 106, - "accuracy": 0.975909090909091 - }, - "0.01": { - "tp": 4174, - "fn": 226, - "accuracy": 0.9486363636363636 - } - }, - "auroc": 0.9789423768939394 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2373, - "fn": 27, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 2340, - "fn": 60, - "accuracy": 0.975 - } - }, - "auroc": 0.9876834201388889 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 6667, - "fn": 133, - "accuracy": 0.9804411764705883 - }, - "0.01": { - "tp": 6514, - "fn": 286, - "accuracy": 0.9579411764705882 - } - }, - "auroc": 0.9820274509803922 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9352697916666667 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9151260416666667 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9251979166666667 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9361125 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9173354166666667 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9267239583333333 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9356911458333332 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9162307291666667 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 744, - "fn": 56, - "accuracy": 0.93 - } - }, - "auroc": 0.9259609375 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9971885416666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9847302083333334 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.990959375 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.975253125 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9813156249999999 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9782843750000001 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9862208333333333 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9830229166666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - }, - "0.01": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - } - }, - "auroc": 0.984621875 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9479187499999999 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9513906249999999 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9496546875 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9428375 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9588343750000001 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9508359375 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.945378125 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9551125 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 784, - "fn": 16, - "accuracy": 0.98 - } - }, - "auroc": 0.9502453125 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9827822916666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9913083333333333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9870453125 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9829458333333334 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.974784375 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9788651041666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9828640625 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9830463541666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - }, - "0.01": { - "tp": 788, - "fn": 12, - "accuracy": 0.985 - } - }, - "auroc": 0.9829552083333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99623125 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.96130625 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9787687500000001 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9763625 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9659041666666668 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9711333333333334 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9862968750000001 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9636052083333334 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - } - }, - "auroc": 0.9749510416666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9417062500000001 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9333677083333334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9375369791666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9356447916666666 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9363020833333333 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9359734375 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9386755208333334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9348348958333333 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 752, - "fn": 48, - "accuracy": 0.94 - } - }, - "auroc": 0.9367552083333333 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9735197916666667 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9735197916666667 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9672822916666667 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9672822916666667 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9704010416666666 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9704010416666666 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9586520833333332 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9586520833333332 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9501322916666667 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9501322916666667 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9543921875000001 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9543921875000001 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 142, - "fn": 58, - "accuracy": 0.71 - } - }, - "auroc": 0.9017083333333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 142, - "fn": 58, - "accuracy": 0.71 - } - }, - "auroc": 0.9017083333333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.89835 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.89835 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 289, - "fn": 111, - "accuracy": 0.7225 - } - }, - "auroc": 0.9000291666666667 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 289, - "fn": 111, - "accuracy": 0.7225 - } - }, - "auroc": 0.9000291666666667 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.8980937499999999 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.8980937499999999 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9179572916666667 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9179572916666667 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 340, - "fn": 60, - "accuracy": 0.85 - } - }, - "auroc": 0.9080255208333333 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 340, - "fn": 60, - "accuracy": 0.85 - } - }, - "auroc": 0.9080255208333333 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9622177083333334 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9622177083333334 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9573406250000001 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9573406250000001 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9597791666666667 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9597791666666667 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2197, - "fn": 3, - "accuracy": 0.9986363636363637 - }, - "0.01": { - "tp": 2040, - "fn": 160, - "accuracy": 0.9272727272727272 - } - }, - "auroc": 0.9541171401515152 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1161, - "fn": 39, - "accuracy": 0.9675 - } - }, - "auroc": 0.9562048611111111 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3395, - "fn": 5, - "accuracy": 0.9985294117647059 - }, - "0.01": { - "tp": 3201, - "fn": 199, - "accuracy": 0.9414705882352942 - } - }, - "auroc": 0.9548539828431372 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2196, - "fn": 4, - "accuracy": 0.9981818181818182 - }, - "0.01": { - "tp": 2087, - "fn": 113, - "accuracy": 0.9486363636363636 - } - }, - "auroc": 0.9491107954545454 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1195, - "fn": 5, - "accuracy": 0.9958333333333333 - }, - "0.01": { - "tp": 1164, - "fn": 36, - "accuracy": 0.97 - } - }, - "auroc": 0.9557460069444446 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3391, - "fn": 9, - "accuracy": 0.9973529411764706 - }, - "0.01": { - "tp": 3251, - "fn": 149, - "accuracy": 0.9561764705882353 - } - }, - "auroc": 0.9514526348039216 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 4393, - "fn": 7, - "accuracy": 0.9984090909090909 - }, - "0.01": { - "tp": 4127, - "fn": 273, - "accuracy": 0.9379545454545455 - } - }, - "auroc": 0.9516139678030304 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2393, - "fn": 7, - "accuracy": 0.9970833333333333 - }, - "0.01": { - "tp": 2325, - "fn": 75, - "accuracy": 0.96875 - } - }, - "auroc": 0.9559754340277777 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 6786, - "fn": 14, - "accuracy": 0.9979411764705882 - }, - "0.01": { - "tp": 6452, - "fn": 348, - "accuracy": 0.9488235294117647 - } - }, - "auroc": 0.9531533088235294 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9745583333333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9800833333333332 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9773208333333333 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9739270833333333 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9706114583333333 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9722692708333333 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9742427083333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9753473958333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9747950520833333 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9720041666666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974791666666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9847416666666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 154, - "fn": 46, - "accuracy": 0.77 - } - }, - "auroc": 0.9041416666666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9798604166666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 346, - "fn": 54, - "accuracy": 0.865 - } - }, - "auroc": 0.9420010416666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 349, - "fn": 51, - "accuracy": 0.8725 - } - }, - "auroc": 0.9380729166666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9886697916666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 789, - "fn": 11, - "accuracy": 0.98625 - }, - "0.01": { - "tp": 740, - "fn": 60, - "accuracy": 0.925 - } - }, - "auroc": 0.9633713541666666 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9851770833333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.988178125 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9866776041666666 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9830489583333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9782052083333335 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9806270833333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9841130208333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9831916666666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.98365234375 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9511020833333332 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9868062500000001 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9689541666666668 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9215635416666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.972521875 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9470427083333334 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9363328125 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9796640624999999 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 787, - "fn": 13, - "accuracy": 0.98375 - }, - "0.01": { - "tp": 757, - "fn": 43, - "accuracy": 0.94625 - } - }, - "auroc": 0.9579984375 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9938958333333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9949187500000001 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9944072916666667 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - } - }, - "auroc": 0.9024291666666666 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.965784375 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 336, - "fn": 64, - "accuracy": 0.84 - } - }, - "auroc": 0.9341067708333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - } - }, - "auroc": 0.9481625 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9803515625000001 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 735, - "fn": 65, - "accuracy": 0.91875 - } - }, - "auroc": 0.9642570312500001 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.981403125 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9849770833333333 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9831901041666666 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9781875 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9809249999999999 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.97955625 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9797953125000001 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9829510416666666 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9813731770833334 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9471958333333335 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9471958333333335 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9349177083333333 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9349177083333333 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - } - }, - "auroc": 0.9410567708333333 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - } - }, - "auroc": 0.9410567708333333 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9184520833333333 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9184520833333333 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 133, - "fn": 67, - "accuracy": 0.665 - } - }, - "auroc": 0.8898052083333334 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 133, - "fn": 67, - "accuracy": 0.665 - } - }, - "auroc": 0.8898052083333334 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 313, - "fn": 87, - "accuracy": 0.7825 - } - }, - "auroc": 0.9041286458333334 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 313, - "fn": 87, - "accuracy": 0.7825 - } - }, - "auroc": 0.9041286458333334 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9935239583333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9935239583333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.989396875 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.989396875 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9914604166666667 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9914604166666667 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9905989583333333 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9905989583333333 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9408343749999999 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9408343749999999 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9657166666666667 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9657166666666667 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9570979166666667 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9570979166666667 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9413447916666666 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9413447916666666 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.9492213541666668 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.9492213541666668 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2189, - "fn": 11, - "accuracy": 0.995 - }, - "0.01": { - "tp": 2135, - "fn": 65, - "accuracy": 0.9704545454545455 - } - }, - "auroc": 0.9695463068181818 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - } - }, - "auroc": 0.9887404513888889 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3387, - "fn": 13, - "accuracy": 0.9961764705882353 - }, - "0.01": { - "tp": 3332, - "fn": 68, - "accuracy": 0.98 - } - }, - "auroc": 0.9763207107843137 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2170, - "fn": 30, - "accuracy": 0.9863636363636363 - }, - "0.01": { - "tp": 1968, - "fn": 232, - "accuracy": 0.8945454545454545 - } - }, - "auroc": 0.941781534090909 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1185, - "fn": 15, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 1160, - "fn": 40, - "accuracy": 0.9666666666666667 - } - }, - "auroc": 0.9746513888888889 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3355, - "fn": 45, - "accuracy": 0.986764705882353 - }, - "0.01": { - "tp": 3128, - "fn": 272, - "accuracy": 0.92 - } - }, - "auroc": 0.9533826593137253 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 4359, - "fn": 41, - "accuracy": 0.9906818181818182 - }, - "0.01": { - "tp": 4103, - "fn": 297, - "accuracy": 0.9325 - } - }, - "auroc": 0.9556639204545454 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2383, - "fn": 17, - "accuracy": 0.9929166666666667 - }, - "0.01": { - "tp": 2357, - "fn": 43, - "accuracy": 0.9820833333333333 - } - }, - "auroc": 0.9816959201388888 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 6742, - "fn": 58, - "accuracy": 0.9914705882352941 - }, - "0.01": { - "tp": 6460, - "fn": 340, - "accuracy": 0.95 - } - }, - "auroc": 0.9648516850490196 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999604166666667 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999802083333333 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.996465625 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9982328125000001 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9982130208333333 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9991065104166666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9948260416666665 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974614583333333 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9961437500000001 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.9249020833333333 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9793916666666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9521468750000001 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9598640625000001 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9884265624999999 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 773, - "fn": 27, - "accuracy": 0.96625 - }, - "0.01": { - "tp": 747, - "fn": 53, - "accuracy": 0.93375 - } - }, - "auroc": 0.9741453125000001 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9936958333333332 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9968468749999999 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998614583333334 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9817239583333334 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9907927083333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999296875 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9877098958333332 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - } - }, - "auroc": 0.9938197916666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9766833333333333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992458333333334 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9879645833333335 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9607072916666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9676885416666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9641979166666668 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9686953125 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9834671875000001 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 783, - "fn": 17, - "accuracy": 0.97875 - }, - "0.01": { - "tp": 761, - "fn": 39, - "accuracy": 0.95125 - } - }, - "auroc": 0.97608125 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9988895833333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9979 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983947916666667 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 159, - "fn": 41, - "accuracy": 0.795 - } - }, - "auroc": 0.9334239583333332 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9643072916666666 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 338, - "fn": 62, - "accuracy": 0.845 - } - }, - "auroc": 0.9488656249999999 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 359, - "fn": 41, - "accuracy": 0.8975 - } - }, - "auroc": 0.9661567708333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9811036458333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 781, - "fn": 19, - "accuracy": 0.97625 - }, - "0.01": { - "tp": 738, - "fn": 62, - "accuracy": 0.9225 - } - }, - "auroc": 0.9736302083333334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99998125 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999895833333334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983229166666666 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9991604166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9991520833333334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995750000000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9874447916666667 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9874447916666667 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9800312499999999 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9800312499999999 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9837380208333334 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9837380208333334 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9725593749999999 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9725593749999999 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - } - }, - "auroc": 0.9337822916666666 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - } - }, - "auroc": 0.9337822916666666 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9531708333333333 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9531708333333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994427083333334 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994427083333334 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997213541666667 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997213541666667 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9898572916666666 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9898572916666666 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9949286458333334 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9949286458333334 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9876239583333333 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9876239583333333 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9703499999999999 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9703499999999999 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - } - }, - "auroc": 0.9789869791666668 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - } - }, - "auroc": 0.9789869791666668 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2188, - "fn": 12, - "accuracy": 0.9945454545454545 - }, - "0.01": { - "tp": 2171, - "fn": 29, - "accuracy": 0.9868181818181818 - } - }, - "auroc": 0.9925475378787878 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9980407986111111 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3386, - "fn": 14, - "accuracy": 0.9958823529411764 - }, - "0.01": { - "tp": 3369, - "fn": 31, - "accuracy": 0.9908823529411764 - } - }, - "auroc": 0.9944863357843138 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2127, - "fn": 73, - "accuracy": 0.9668181818181818 - }, - "0.01": { - "tp": 2037, - "fn": 163, - "accuracy": 0.9259090909090909 - } - }, - "auroc": 0.9720323863636362 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1181, - "fn": 19, - "accuracy": 0.9841666666666666 - }, - "0.01": { - "tp": 1149, - "fn": 51, - "accuracy": 0.9575 - } - }, - "auroc": 0.9813166666666667 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3308, - "fn": 92, - "accuracy": 0.9729411764705882 - }, - "0.01": { - "tp": 3186, - "fn": 214, - "accuracy": 0.9370588235294117 - } - }, - "auroc": 0.9753091911764706 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 4315, - "fn": 85, - "accuracy": 0.9806818181818182 - }, - "0.01": { - "tp": 4208, - "fn": 192, - "accuracy": 0.9563636363636364 - } - }, - "auroc": 0.9822899621212121 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2379, - "fn": 21, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 2347, - "fn": 53, - "accuracy": 0.9779166666666667 - } - }, - "auroc": 0.9896787326388888 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 6694, - "fn": 106, - "accuracy": 0.9844117647058823 - }, - "0.01": { - "tp": 6555, - "fn": 245, - "accuracy": 0.9639705882352941 - } - }, - "auroc": 0.9848977634803922 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999708333333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999854166666666 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9951895833333333 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9975947916666666 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9975802083333334 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9987901041666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9943510416666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974791666666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9959151041666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - }, - "0.01": { - "tp": 146, - "fn": 54, - "accuracy": 0.73 - } - }, - "auroc": 0.9041854166666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9798604166666666 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - }, - "0.01": { - "tp": 338, - "fn": 62, - "accuracy": 0.845 - } - }, - "auroc": 0.9420229166666668 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 343, - "fn": 57, - "accuracy": 0.8575 - } - }, - "auroc": 0.9492682291666668 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9886697916666667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - }, - "0.01": { - "tp": 734, - "fn": 66, - "accuracy": 0.9175 - } - }, - "auroc": 0.9689690104166667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.992946875 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9964718749999999 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999729166666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9792645833333333 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.98961875 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999848958333334 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9861057291666667 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9930453124999999 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9763072916666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992614583333334 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9877843749999999 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9487760416666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9568343750000001 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 359, - "fn": 41, - "accuracy": 0.8975 - } - }, - "auroc": 0.9528052083333333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9625416666666666 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9780479166666667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 778, - "fn": 22, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 755, - "fn": 45, - "accuracy": 0.94375 - } - }, - "auroc": 0.9702947916666667 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9988729166666667 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.997546875 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9982098958333334 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - } - }, - "auroc": 0.9126593749999999 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.957565625 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 326, - "fn": 74, - "accuracy": 0.815 - } - }, - "auroc": 0.9351125 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - } - }, - "auroc": 0.9557661458333333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.97755625 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 726, - "fn": 74, - "accuracy": 0.9075 - } - }, - "auroc": 0.9666611979166667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999802083333332 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999890625 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9972989583333334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986484375 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666667 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986395833333334 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99931875 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.988271875 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.988271875 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9779406250000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9779406250000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9831062500000001 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9831062500000001 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.966665625 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.966665625 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9258541666666666 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9258541666666666 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9462598958333335 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9462598958333335 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999984375000001 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999984375000001 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 1.0 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9900260416666666 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9900260416666666 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9950130208333333 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9950130208333333 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9844625 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9844625 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9693427083333335 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9693427083333335 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9769026041666667 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9769026041666667 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2188, - "fn": 12, - "accuracy": 0.9945454545454545 - }, - "0.01": { - "tp": 2167, - "fn": 33, - "accuracy": 0.985 - } - }, - "auroc": 0.9917205492424243 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9978642361111111 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3386, - "fn": 14, - "accuracy": 0.9958823529411764 - }, - "0.01": { - "tp": 3365, - "fn": 35, - "accuracy": 0.9897058823529412 - } - }, - "auroc": 0.9938889093137255 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2107, - "fn": 93, - "accuracy": 0.9577272727272728 - }, - "0.01": { - "tp": 2009, - "fn": 191, - "accuracy": 0.9131818181818182 - } - }, - "auroc": 0.9662501893939395 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1177, - "fn": 23, - "accuracy": 0.9808333333333333 - }, - "0.01": { - "tp": 1143, - "fn": 57, - "accuracy": 0.9525 - } - }, - "auroc": 0.9776689236111111 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3284, - "fn": 116, - "accuracy": 0.9658823529411765 - }, - "0.01": { - "tp": 3152, - "fn": 248, - "accuracy": 0.9270588235294117 - } - }, - "auroc": 0.970280330882353 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 4295, - "fn": 105, - "accuracy": 0.9761363636363637 - }, - "0.01": { - "tp": 4176, - "fn": 224, - "accuracy": 0.9490909090909091 - } - }, - "auroc": 0.9789853693181818 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2375, - "fn": 25, - "accuracy": 0.9895833333333334 - }, - "0.01": { - "tp": 2341, - "fn": 59, - "accuracy": 0.9754166666666667 - } - }, - "auroc": 0.987766579861111 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 6670, - "fn": 130, - "accuracy": 0.9808823529411764 - }, - "0.01": { - "tp": 6517, - "fn": 283, - "accuracy": 0.9583823529411765 - } - }, - "auroc": 0.9820846200980393 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2389, - "fn": 11, - "accuracy": 0.9954166666666666 - } - }, - "auroc": 0.9923765625000001 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2381, - "fn": 19, - "accuracy": 0.9920833333333333 - } - }, - "auroc": 0.990455295138889 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - }, - "0.01": { - "tp": 4770, - "fn": 30, - "accuracy": 0.99375 - } - }, - "auroc": 0.9914159288194444 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2388, - "fn": 12, - "accuracy": 0.995 - } - }, - "auroc": 0.9922014756944444 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 2356, - "fn": 44, - "accuracy": 0.9816666666666667 - } - }, - "auroc": 0.9847166666666667 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4796, - "fn": 4, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 4744, - "fn": 56, - "accuracy": 0.9883333333333333 - } - }, - "auroc": 0.9884590711805555 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4777, - "fn": 23, - "accuracy": 0.9952083333333334 - } - }, - "auroc": 0.9922890190972222 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4795, - "fn": 5, - "accuracy": 0.9989583333333333 - }, - "0.01": { - "tp": 4737, - "fn": 63, - "accuracy": 0.986875 - } - }, - "auroc": 0.9875859809027777 - }, - { - "domain": "books", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9595, - "fn": 5, - "accuracy": 0.9994791666666667 - }, - "0.01": { - "tp": 9514, - "fn": 86, - "accuracy": 0.9910416666666667 - } - }, - "auroc": 0.9899374999999999 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2374, - "fn": 26, - "accuracy": 0.9891666666666666 - } - }, - "auroc": 0.9931652777777777 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2371, - "fn": 29, - "accuracy": 0.9879166666666667 - }, - "0.01": { - "tp": 2315, - "fn": 85, - "accuracy": 0.9645833333333333 - } - }, - "auroc": 0.9869315104166667 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4771, - "fn": 29, - "accuracy": 0.9939583333333334 - }, - "0.01": { - "tp": 4689, - "fn": 111, - "accuracy": 0.976875 - } - }, - "auroc": 0.9900483940972222 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2186, - "fn": 214, - "accuracy": 0.9108333333333334 - }, - "0.01": { - "tp": 1917, - "fn": 483, - "accuracy": 0.79875 - } - }, - "auroc": 0.9246156249999999 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2309, - "fn": 91, - "accuracy": 0.9620833333333333 - }, - "0.01": { - "tp": 2215, - "fn": 185, - "accuracy": 0.9229166666666667 - } - }, - "auroc": 0.9676954861111111 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4495, - "fn": 305, - "accuracy": 0.9364583333333333 - }, - "0.01": { - "tp": 4132, - "fn": 668, - "accuracy": 0.8608333333333333 - } - }, - "auroc": 0.9461555555555556 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4586, - "fn": 214, - "accuracy": 0.9554166666666667 - }, - "0.01": { - "tp": 4291, - "fn": 509, - "accuracy": 0.8939583333333333 - } - }, - "auroc": 0.958890451388889 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4680, - "fn": 120, - "accuracy": 0.975 - }, - "0.01": { - "tp": 4530, - "fn": 270, - "accuracy": 0.94375 - } - }, - "auroc": 0.9773134982638889 - }, - { - "domain": "books", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9266, - "fn": 334, - "accuracy": 0.9652083333333333 - }, - "0.01": { - "tp": 8821, - "fn": 779, - "accuracy": 0.9188541666666666 - } - }, - "auroc": 0.968101974826389 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2385, - "fn": 15, - "accuracy": 0.99375 - } - }, - "auroc": 0.9922638888888888 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2382, - "fn": 18, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 2347, - "fn": 53, - "accuracy": 0.9779166666666667 - } - }, - "auroc": 0.9840703993055556 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4781, - "fn": 19, - "accuracy": 0.9960416666666667 - }, - "0.01": { - "tp": 4732, - "fn": 68, - "accuracy": 0.9858333333333333 - } - }, - "auroc": 0.9881671440972222 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2382, - "fn": 18, - "accuracy": 0.9925 - } - }, - "auroc": 0.9914800347222222 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2379, - "fn": 21, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 2317, - "fn": 83, - "accuracy": 0.9654166666666667 - } - }, - "auroc": 0.9713291666666666 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4778, - "fn": 22, - "accuracy": 0.9954166666666666 - }, - "0.01": { - "tp": 4699, - "fn": 101, - "accuracy": 0.9789583333333334 - } - }, - "auroc": 0.9814046006944444 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4798, - "fn": 2, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 4767, - "fn": 33, - "accuracy": 0.993125 - } - }, - "auroc": 0.9918719618055556 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4761, - "fn": 39, - "accuracy": 0.991875 - }, - "0.01": { - "tp": 4664, - "fn": 136, - "accuracy": 0.9716666666666667 - } - }, - "auroc": 0.9776997829861112 - }, - { - "domain": "books", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9559, - "fn": 41, - "accuracy": 0.9957291666666667 - }, - "0.01": { - "tp": 9431, - "fn": 169, - "accuracy": 0.9823958333333334 - } - }, - "auroc": 0.9847858723958334 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2384, - "fn": 16, - "accuracy": 0.9933333333333333 - }, - "0.01": { - "tp": 2343, - "fn": 57, - "accuracy": 0.97625 - } - }, - "auroc": 0.9763159722222222 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2395, - "fn": 5, - "accuracy": 0.9979166666666667 - } - }, - "auroc": 0.9961618923611112 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4783, - "fn": 17, - "accuracy": 0.9964583333333333 - }, - "0.01": { - "tp": 4738, - "fn": 62, - "accuracy": 0.9870833333333333 - } - }, - "auroc": 0.9862389322916667 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2296, - "fn": 104, - "accuracy": 0.9566666666666667 - }, - "0.01": { - "tp": 2176, - "fn": 224, - "accuracy": 0.9066666666666666 - } - }, - "auroc": 0.9541905381944444 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2278, - "fn": 122, - "accuracy": 0.9491666666666667 - }, - "0.01": { - "tp": 2140, - "fn": 260, - "accuracy": 0.8916666666666667 - } - }, - "auroc": 0.9571289930555555 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4574, - "fn": 226, - "accuracy": 0.9529166666666666 - }, - "0.01": { - "tp": 4316, - "fn": 484, - "accuracy": 0.8991666666666667 - } - }, - "auroc": 0.9556597656250001 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4680, - "fn": 120, - "accuracy": 0.975 - }, - "0.01": { - "tp": 4519, - "fn": 281, - "accuracy": 0.9414583333333333 - } - }, - "auroc": 0.9652532552083333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4677, - "fn": 123, - "accuracy": 0.974375 - }, - "0.01": { - "tp": 4535, - "fn": 265, - "accuracy": 0.9447916666666667 - } - }, - "auroc": 0.9766454427083333 - }, - { - "domain": "books", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9357, - "fn": 243, - "accuracy": 0.9746875 - }, - "0.01": { - "tp": 9054, - "fn": 546, - "accuracy": 0.943125 - } - }, - "auroc": 0.9709493489583334 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981747395833334 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2394, - "fn": 6, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 2368, - "fn": 32, - "accuracy": 0.9866666666666667 - } - }, - "auroc": 0.9908621527777779 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4794, - "fn": 6, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 4768, - "fn": 32, - "accuracy": 0.9933333333333333 - } - }, - "auroc": 0.9945184461805554 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2248, - "fn": 152, - "accuracy": 0.9366666666666666 - }, - "0.01": { - "tp": 1944, - "fn": 456, - "accuracy": 0.81 - } - }, - "auroc": 0.9289573784722224 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2256, - "fn": 144, - "accuracy": 0.94 - }, - "0.01": { - "tp": 2070, - "fn": 330, - "accuracy": 0.8625 - } - }, - "auroc": 0.9492694444444445 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4504, - "fn": 296, - "accuracy": 0.9383333333333334 - }, - "0.01": { - "tp": 4014, - "fn": 786, - "accuracy": 0.83625 - } - }, - "auroc": 0.9391134114583333 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4648, - "fn": 152, - "accuracy": 0.9683333333333334 - }, - "0.01": { - "tp": 4344, - "fn": 456, - "accuracy": 0.905 - } - }, - "auroc": 0.9635660590277778 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4650, - "fn": 150, - "accuracy": 0.96875 - }, - "0.01": { - "tp": 4438, - "fn": 362, - "accuracy": 0.9245833333333333 - } - }, - "auroc": 0.970065798611111 - }, - { - "domain": "books", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9298, - "fn": 302, - "accuracy": 0.9685416666666666 - }, - "0.01": { - "tp": 8782, - "fn": 818, - "accuracy": 0.9147916666666667 - } - }, - "auroc": 0.9668159288194444 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2382, - "fn": 18, - "accuracy": 0.9925 - } - }, - "auroc": 0.9930993923611111 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2387, - "fn": 13, - "accuracy": 0.9945833333333334 - } - }, - "auroc": 0.9923389756944445 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4769, - "fn": 31, - "accuracy": 0.9935416666666667 - } - }, - "auroc": 0.9927191840277777 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2387, - "fn": 13, - "accuracy": 0.9945833333333334 - } - }, - "auroc": 0.992000607638889 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 2367, - "fn": 33, - "accuracy": 0.98625 - } - }, - "auroc": 0.9875665798611111 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4797, - "fn": 3, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 4754, - "fn": 46, - "accuracy": 0.9904166666666666 - } - }, - "auroc": 0.98978359375 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - }, - "0.01": { - "tp": 4769, - "fn": 31, - "accuracy": 0.9935416666666667 - } - }, - "auroc": 0.9925499999999999 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4798, - "fn": 2, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 4754, - "fn": 46, - "accuracy": 0.9904166666666666 - } - }, - "auroc": 0.9899527777777778 - }, - { - "domain": "books", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9597, - "fn": 3, - "accuracy": 0.9996875 - }, - "0.01": { - "tp": 9523, - "fn": 77, - "accuracy": 0.9919791666666666 - } - }, - "auroc": 0.9912513888888889 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2364, - "fn": 36, - "accuracy": 0.985 - }, - "0.01": { - "tp": 2298, - "fn": 102, - "accuracy": 0.9575 - } - }, - "auroc": 0.9785522569444445 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2364, - "fn": 36, - "accuracy": 0.985 - }, - "0.01": { - "tp": 2298, - "fn": 102, - "accuracy": 0.9575 - } - }, - "auroc": 0.9785522569444445 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2344, - "fn": 56, - "accuracy": 0.9766666666666667 - }, - "0.01": { - "tp": 2243, - "fn": 157, - "accuracy": 0.9345833333333333 - } - }, - "auroc": 0.9691095486111112 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2344, - "fn": 56, - "accuracy": 0.9766666666666667 - }, - "0.01": { - "tp": 2243, - "fn": 157, - "accuracy": 0.9345833333333333 - } - }, - "auroc": 0.9691095486111112 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4708, - "fn": 92, - "accuracy": 0.9808333333333333 - }, - "0.01": { - "tp": 4541, - "fn": 259, - "accuracy": 0.9460416666666667 - } - }, - "auroc": 0.9738309027777778 - }, - { - "domain": "books", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4708, - "fn": 92, - "accuracy": 0.9808333333333333 - }, - "0.01": { - "tp": 4541, - "fn": 259, - "accuracy": 0.9460416666666667 - } - }, - "auroc": 0.9738309027777778 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2344, - "fn": 56, - "accuracy": 0.9766666666666667 - }, - "0.01": { - "tp": 2236, - "fn": 164, - "accuracy": 0.9316666666666666 - } - }, - "auroc": 0.9612210069444445 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2344, - "fn": 56, - "accuracy": 0.9766666666666667 - }, - "0.01": { - "tp": 2236, - "fn": 164, - "accuracy": 0.9316666666666666 - } - }, - "auroc": 0.9612210069444445 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2204, - "fn": 196, - "accuracy": 0.9183333333333333 - }, - "0.01": { - "tp": 1986, - "fn": 414, - "accuracy": 0.8275 - } - }, - "auroc": 0.9279925347222222 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2204, - "fn": 196, - "accuracy": 0.9183333333333333 - }, - "0.01": { - "tp": 1986, - "fn": 414, - "accuracy": 0.8275 - } - }, - "auroc": 0.9279925347222222 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4548, - "fn": 252, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 4222, - "fn": 578, - "accuracy": 0.8795833333333334 - } - }, - "auroc": 0.9446067708333333 - }, - { - "domain": "books", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4548, - "fn": 252, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 4222, - "fn": 578, - "accuracy": 0.8795833333333334 - } - }, - "auroc": 0.9446067708333333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2342, - "fn": 58, - "accuracy": 0.9758333333333333 - } - }, - "auroc": 0.9908251736111111 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2342, - "fn": 58, - "accuracy": 0.9758333333333333 - } - }, - "auroc": 0.9908251736111111 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2343, - "fn": 57, - "accuracy": 0.97625 - } - }, - "auroc": 0.9896122395833333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2343, - "fn": 57, - "accuracy": 0.97625 - } - }, - "auroc": 0.9896122395833333 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - }, - "0.01": { - "tp": 4685, - "fn": 115, - "accuracy": 0.9760416666666667 - } - }, - "auroc": 0.9902187065972222 - }, - { - "domain": "books", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - }, - "0.01": { - "tp": 4685, - "fn": 115, - "accuracy": 0.9760416666666667 - } - }, - "auroc": 0.9902187065972222 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2345, - "fn": 55, - "accuracy": 0.9770833333333333 - } - }, - "auroc": 0.9902345486111112 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2345, - "fn": 55, - "accuracy": 0.9770833333333333 - } - }, - "auroc": 0.9902345486111112 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2397, - "fn": 3, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 2356, - "fn": 44, - "accuracy": 0.9816666666666667 - } - }, - "auroc": 0.9777543402777777 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2397, - "fn": 3, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 2356, - "fn": 44, - "accuracy": 0.9816666666666667 - } - }, - "auroc": 0.9777543402777777 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4796, - "fn": 4, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 4701, - "fn": 99, - "accuracy": 0.979375 - } - }, - "auroc": 0.9839944444444444 - }, - { - "domain": "books", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4796, - "fn": 4, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 4701, - "fn": 99, - "accuracy": 0.979375 - } - }, - "auroc": 0.9839944444444444 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2351, - "fn": 49, - "accuracy": 0.9795833333333334 - }, - "0.01": { - "tp": 2266, - "fn": 134, - "accuracy": 0.9441666666666667 - } - }, - "auroc": 0.9780388888888889 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2351, - "fn": 49, - "accuracy": 0.9795833333333334 - }, - "0.01": { - "tp": 2266, - "fn": 134, - "accuracy": 0.9441666666666667 - } - }, - "auroc": 0.9780388888888889 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2304, - "fn": 96, - "accuracy": 0.96 - }, - "0.01": { - "tp": 2177, - "fn": 223, - "accuracy": 0.9070833333333334 - } - }, - "auroc": 0.9642591145833332 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2304, - "fn": 96, - "accuracy": 0.96 - }, - "0.01": { - "tp": 2177, - "fn": 223, - "accuracy": 0.9070833333333334 - } - }, - "auroc": 0.9642591145833332 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4655, - "fn": 145, - "accuracy": 0.9697916666666667 - }, - "0.01": { - "tp": 4443, - "fn": 357, - "accuracy": 0.925625 - } - }, - "auroc": 0.9711490017361111 - }, - { - "domain": "books", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4655, - "fn": 145, - "accuracy": 0.9697916666666667 - }, - "0.01": { - "tp": 4443, - "fn": 357, - "accuracy": 0.925625 - } - }, - "auroc": 0.9711490017361111 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 26241, - "fn": 159, - "accuracy": 0.9939772727272728 - }, - "0.01": { - "tp": 25760, - "fn": 640, - "accuracy": 0.9757575757575757 - } - }, - "auroc": 0.985842518939394 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 14345, - "fn": 55, - "accuracy": 0.9961805555555555 - }, - "0.01": { - "tp": 14193, - "fn": 207, - "accuracy": 0.985625 - } - }, - "auroc": 0.9901367042824074 - }, - { - "domain": "books", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 40586, - "fn": 214, - "accuracy": 0.9947549019607843 - }, - "0.01": { - "tp": 39953, - "fn": 847, - "accuracy": 0.9792401960784314 - } - }, - "auroc": 0.9873581137663399 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 25576, - "fn": 824, - "accuracy": 0.9687878787878788 - }, - "0.01": { - "tp": 24299, - "fn": 2101, - "accuracy": 0.9204166666666667 - } - }, - "auroc": 0.9647430397727272 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 14016, - "fn": 384, - "accuracy": 0.9733333333333334 - }, - "0.01": { - "tp": 13465, - "fn": 935, - "accuracy": 0.9350694444444444 - } - }, - "auroc": 0.9696177228009258 - }, - { - "domain": "books", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 39592, - "fn": 1208, - "accuracy": 0.970392156862745 - }, - "0.01": { - "tp": 37764, - "fn": 3036, - "accuracy": 0.9255882352941176 - } - }, - "auroc": 0.9664635161356209 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 51817, - "fn": 983, - "accuracy": 0.9813825757575757 - }, - "0.01": { - "tp": 50059, - "fn": 2741, - "accuracy": 0.9480871212121212 - } - }, - "auroc": 0.9752927793560606 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 28361, - "fn": 439, - "accuracy": 0.9847569444444444 - }, - "0.01": { - "tp": 27658, - "fn": 1142, - "accuracy": 0.9603472222222222 - } - }, - "auroc": 0.9798772135416666 - }, - { - "domain": "books", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 80178, - "fn": 1422, - "accuracy": 0.9825735294117647 - }, - "0.01": { - "tp": 77717, - "fn": 3883, - "accuracy": 0.9524142156862745 - } - }, - "auroc": 0.9769108149509804 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998489583333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999156250000001 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999791666666666 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9967677083333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9983734375 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999807291666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9983083333333334 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.99914453125 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9817760416666668 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.97351875 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9776473958333334 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - }, - "0.01": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - } - }, - "auroc": 0.8585822916666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9490958333333332 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 349, - "fn": 51, - "accuracy": 0.8725 - }, - "0.01": { - "tp": 317, - "fn": 83, - "accuracy": 0.7925 - } - }, - "auroc": 0.9038390625 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9201791666666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9613072916666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 741, - "fn": 59, - "accuracy": 0.92625 - }, - "0.01": { - "tp": 703, - "fn": 97, - "accuracy": 0.87875 - } - }, - "auroc": 0.9407432291666668 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997510416666666 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9979958333333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9988734375 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9970187500000001 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9781083333333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9875635416666667 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983848958333335 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9880520833333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - } - }, - "auroc": 0.9932184895833333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.95169375 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9918156250000001 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9717546875 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 116, - "fn": 84, - "accuracy": 0.58 - } - }, - "auroc": 0.8512177083333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9464479166666666 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 307, - "fn": 93, - "accuracy": 0.7675 - } - }, - "auroc": 0.8988328125 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 302, - "fn": 98, - "accuracy": 0.755 - } - }, - "auroc": 0.9014557291666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9691317708333334 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - }, - "0.01": { - "tp": 691, - "fn": 109, - "accuracy": 0.86375 - } - }, - "auroc": 0.93529375 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.98485 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9401322916666667 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9624911458333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 129, - "fn": 71, - "accuracy": 0.645 - }, - "0.01": { - "tp": 86, - "fn": 114, - "accuracy": 0.43 - } - }, - "auroc": 0.7668166666666666 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9213927083333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 321, - "fn": 79, - "accuracy": 0.8025 - }, - "0.01": { - "tp": 265, - "fn": 135, - "accuracy": 0.6625 - } - }, - "auroc": 0.8441046875 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 329, - "fn": 71, - "accuracy": 0.8225 - }, - "0.01": { - "tp": 283, - "fn": 117, - "accuracy": 0.7075 - } - }, - "auroc": 0.8758333333333335 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - } - }, - "auroc": 0.9307625 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 716, - "fn": 84, - "accuracy": 0.895 - }, - "0.01": { - "tp": 646, - "fn": 154, - "accuracy": 0.8075 - } - }, - "auroc": 0.9032979166666666 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99995625 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999884375 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999203125 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9948614583333335 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9879322916666666 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9913968750000001 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9974088541666667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9939083333333334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9956585937500001 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.977653125 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.977653125 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9677802083333333 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9677802083333333 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9727166666666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9727166666666667 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - } - }, - "auroc": 0.9237114583333332 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - } - }, - "auroc": 0.9237114583333332 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 139, - "fn": 61, - "accuracy": 0.695 - } - }, - "auroc": 0.8780041666666667 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 139, - "fn": 61, - "accuracy": 0.695 - } - }, - "auroc": 0.8780041666666667 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 309, - "fn": 91, - "accuracy": 0.7725 - } - }, - "auroc": 0.9008578125 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 309, - "fn": 91, - "accuracy": 0.7725 - } - }, - "auroc": 0.9008578125 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99998125 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99998125 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999604166666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999604166666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999708333333334 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999708333333334 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999385416666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999385416666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9894635416666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9894635416666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9947010416666667 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9947010416666667 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.945846875 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.945846875 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9385531250000001 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9385531250000001 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9422 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9422 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2174, - "fn": 26, - "accuracy": 0.9881818181818182 - }, - "0.01": { - "tp": 2121, - "fn": 79, - "accuracy": 0.9640909090909091 - } - }, - "auroc": 0.9786491477272727 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1187, - "fn": 13, - "accuracy": 0.9891666666666666 - }, - "0.01": { - "tp": 1170, - "fn": 30, - "accuracy": 0.975 - } - }, - "auroc": 0.9838659722222223 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3361, - "fn": 39, - "accuracy": 0.9885294117647059 - }, - "0.01": { - "tp": 3291, - "fn": 109, - "accuracy": 0.9679411764705882 - } - }, - "auroc": 0.9804903799019608 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2034, - "fn": 166, - "accuracy": 0.9245454545454546 - }, - "0.01": { - "tp": 1840, - "fn": 360, - "accuracy": 0.8363636363636363 - } - }, - "auroc": 0.9311125 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1174, - "fn": 26, - "accuracy": 0.9783333333333334 - }, - "0.01": { - "tp": 1146, - "fn": 54, - "accuracy": 0.955 - } - }, - "auroc": 0.9632907986111111 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3208, - "fn": 192, - "accuracy": 0.9435294117647058 - }, - "0.01": { - "tp": 2986, - "fn": 414, - "accuracy": 0.8782352941176471 - } - }, - "auroc": 0.9424695465686275 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 4208, - "fn": 192, - "accuracy": 0.9563636363636364 - }, - "0.01": { - "tp": 3961, - "fn": 439, - "accuracy": 0.9002272727272728 - } - }, - "auroc": 0.9548808238636364 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2361, - "fn": 39, - "accuracy": 0.98375 - }, - "0.01": { - "tp": 2316, - "fn": 84, - "accuracy": 0.965 - } - }, - "auroc": 0.9735783854166666 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 6569, - "fn": 231, - "accuracy": 0.9660294117647059 - }, - "0.01": { - "tp": 6277, - "fn": 523, - "accuracy": 0.9230882352941177 - } - }, - "auroc": 0.961479963235294 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998489583333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999156250000001 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999791666666666 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9967677083333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9983734375 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999807291666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9983083333333334 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.99914453125 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9817760416666668 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.97351875 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9776473958333334 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - }, - "0.01": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - } - }, - "auroc": 0.8585822916666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9490958333333332 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 349, - "fn": 51, - "accuracy": 0.8725 - }, - "0.01": { - "tp": 317, - "fn": 83, - "accuracy": 0.7925 - } - }, - "auroc": 0.9038390625 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9201791666666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9613072916666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 741, - "fn": 59, - "accuracy": 0.92625 - }, - "0.01": { - "tp": 703, - "fn": 97, - "accuracy": 0.87875 - } - }, - "auroc": 0.9407432291666668 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997510416666666 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9979958333333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9988734375 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9970187500000001 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9781083333333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9875635416666667 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983848958333335 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9880520833333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - } - }, - "auroc": 0.9932184895833333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.95169375 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9918156250000001 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9717546875 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 116, - "fn": 84, - "accuracy": 0.58 - } - }, - "auroc": 0.8512177083333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9464479166666666 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 307, - "fn": 93, - "accuracy": 0.7675 - } - }, - "auroc": 0.8988328125 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 302, - "fn": 98, - "accuracy": 0.755 - } - }, - "auroc": 0.9014557291666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9691317708333334 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - }, - "0.01": { - "tp": 691, - "fn": 109, - "accuracy": 0.86375 - } - }, - "auroc": 0.93529375 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.98485 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9401322916666667 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9624911458333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 129, - "fn": 71, - "accuracy": 0.645 - }, - "0.01": { - "tp": 86, - "fn": 114, - "accuracy": 0.43 - } - }, - "auroc": 0.7668166666666666 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9213927083333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 321, - "fn": 79, - "accuracy": 0.8025 - }, - "0.01": { - "tp": 265, - "fn": 135, - "accuracy": 0.6625 - } - }, - "auroc": 0.8441046875 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 329, - "fn": 71, - "accuracy": 0.8225 - }, - "0.01": { - "tp": 283, - "fn": 117, - "accuracy": 0.7075 - } - }, - "auroc": 0.8758333333333335 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - } - }, - "auroc": 0.9307625 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 716, - "fn": 84, - "accuracy": 0.895 - }, - "0.01": { - "tp": 646, - "fn": 154, - "accuracy": 0.8075 - } - }, - "auroc": 0.9032979166666666 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99995625 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999884375 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999203125 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9948614583333335 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9879322916666666 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9913968750000001 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9974088541666667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9939083333333334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9956585937500001 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.977653125 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.977653125 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9677802083333333 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9677802083333333 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9727166666666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9727166666666667 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - } - }, - "auroc": 0.9237114583333332 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - } - }, - "auroc": 0.9237114583333332 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 139, - "fn": 61, - "accuracy": 0.695 - } - }, - "auroc": 0.8780041666666667 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 139, - "fn": 61, - "accuracy": 0.695 - } - }, - "auroc": 0.8780041666666667 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 309, - "fn": 91, - "accuracy": 0.7725 - } - }, - "auroc": 0.9008578125 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 309, - "fn": 91, - "accuracy": 0.7725 - } - }, - "auroc": 0.9008578125 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99998125 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99998125 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999604166666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999604166666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999708333333334 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999708333333334 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999385416666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999385416666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9894635416666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9894635416666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9947010416666667 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9947010416666667 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.945846875 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.945846875 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9385531250000001 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9385531250000001 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9422 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9422 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2174, - "fn": 26, - "accuracy": 0.9881818181818182 - }, - "0.01": { - "tp": 2121, - "fn": 79, - "accuracy": 0.9640909090909091 - } - }, - "auroc": 0.9786491477272727 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1187, - "fn": 13, - "accuracy": 0.9891666666666666 - }, - "0.01": { - "tp": 1170, - "fn": 30, - "accuracy": 0.975 - } - }, - "auroc": 0.9838659722222223 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3361, - "fn": 39, - "accuracy": 0.9885294117647059 - }, - "0.01": { - "tp": 3291, - "fn": 109, - "accuracy": 0.9679411764705882 - } - }, - "auroc": 0.9804903799019608 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2034, - "fn": 166, - "accuracy": 0.9245454545454546 - }, - "0.01": { - "tp": 1840, - "fn": 360, - "accuracy": 0.8363636363636363 - } - }, - "auroc": 0.9311125 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1174, - "fn": 26, - "accuracy": 0.9783333333333334 - }, - "0.01": { - "tp": 1146, - "fn": 54, - "accuracy": 0.955 - } - }, - "auroc": 0.9632907986111111 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3208, - "fn": 192, - "accuracy": 0.9435294117647058 - }, - "0.01": { - "tp": 2986, - "fn": 414, - "accuracy": 0.8782352941176471 - } - }, - "auroc": 0.9424695465686275 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 4208, - "fn": 192, - "accuracy": 0.9563636363636364 - }, - "0.01": { - "tp": 3961, - "fn": 439, - "accuracy": 0.9002272727272728 - } - }, - "auroc": 0.9548808238636364 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2361, - "fn": 39, - "accuracy": 0.98375 - }, - "0.01": { - "tp": 2316, - "fn": 84, - "accuracy": 0.965 - } - }, - "auroc": 0.9735783854166666 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 6569, - "fn": 231, - "accuracy": 0.9660294117647059 - }, - "0.01": { - "tp": 6277, - "fn": 523, - "accuracy": 0.9230882352941177 - } - }, - "auroc": 0.961479963235294 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999770833333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993593749999999 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996682291666666 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994635416666666 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.996840625 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981520833333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997203125 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99891015625 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9972083333333334 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9720072916666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9846078125 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9508791666666666 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9459614583333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9484203125 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.97404375 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - } - }, - "auroc": 0.958984375 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 779, - "fn": 21, - "accuracy": 0.97375 - }, - "0.01": { - "tp": 765, - "fn": 35, - "accuracy": 0.95625 - } - }, - "auroc": 0.9665140625 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996833333333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9972625 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9984729166666666 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9970729166666665 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9773093749999999 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9871911458333333 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983781250000001 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9872859375 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.99283203125 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.984378125 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9977343750000001 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9910562500000001 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9536510416666668 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9801833333333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9669171875000001 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9690145833333335 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9889588541666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 784, - "fn": 16, - "accuracy": 0.98 - } - }, - "auroc": 0.9789867187500001 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9944010416666667 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9899958333333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9921984374999999 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9161958333333333 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9568145833333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9365052083333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9552984375 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9734052083333333 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - }, - "0.01": { - "tp": 777, - "fn": 23, - "accuracy": 0.97125 - } - }, - "auroc": 0.9643518229166667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999177083333334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993677083333333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996427083333334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.99614375 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.991675 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9939093750000001 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9980307291666667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9955213541666665 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9967760416666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9894052083333333 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9894052083333333 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.982903125 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.982903125 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9861541666666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9861541666666667 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9705458333333334 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9705458333333334 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9411364583333335 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9411364583333335 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9558411458333332 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9558411458333332 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999239583333334 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999239583333334 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993624999999999 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993624999999999 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996432291666666 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996432291666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998864583333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998864583333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9916427083333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9916427083333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9957645833333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9957645833333333 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9778125 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9778125 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9682312500000001 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9682312500000001 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9730218749999999 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9730218749999999 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2197, - "fn": 3, - "accuracy": 0.9986363636363637 - }, - "0.01": { - "tp": 2187, - "fn": 13, - "accuracy": 0.9940909090909091 - } - }, - "auroc": 0.9921035984848484 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1193, - "fn": 7, - "accuracy": 0.9941666666666666 - }, - "0.01": { - "tp": 1189, - "fn": 11, - "accuracy": 0.9908333333333333 - } - }, - "auroc": 0.9926211805555556 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3390, - "fn": 10, - "accuracy": 0.9970588235294118 - }, - "0.01": { - "tp": 3376, - "fn": 24, - "accuracy": 0.9929411764705882 - } - }, - "auroc": 0.9922862745098039 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2192, - "fn": 8, - "accuracy": 0.9963636363636363 - }, - "0.01": { - "tp": 2155, - "fn": 45, - "accuracy": 0.9795454545454545 - } - }, - "auroc": 0.9724256628787878 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1181, - "fn": 19, - "accuracy": 0.9841666666666666 - }, - "0.01": { - "tp": 1172, - "fn": 28, - "accuracy": 0.9766666666666667 - } - }, - "auroc": 0.9747973958333334 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3373, - "fn": 27, - "accuracy": 0.9920588235294118 - }, - "0.01": { - "tp": 3327, - "fn": 73, - "accuracy": 0.9785294117647059 - } - }, - "auroc": 0.9732627450980392 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 4389, - "fn": 11, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 4342, - "fn": 58, - "accuracy": 0.9868181818181818 - } - }, - "auroc": 0.9822646306818181 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2374, - "fn": 26, - "accuracy": 0.9891666666666666 - }, - "0.01": { - "tp": 2361, - "fn": 39, - "accuracy": 0.98375 - } - }, - "auroc": 0.9837092881944445 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 6763, - "fn": 37, - "accuracy": 0.9945588235294117 - }, - "0.01": { - "tp": 6703, - "fn": 97, - "accuracy": 0.985735294117647 - } - }, - "auroc": 0.9827745098039216 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999760416666668 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999384375 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996802083333334 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9922166666666666 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9960583333333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999380208333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9958005208333334 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9978692708333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.980125 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.96538125 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9727531250000001 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - }, - "0.01": { - "tp": 141, - "fn": 59, - "accuracy": 0.705 - } - }, - "auroc": 0.8697270833333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9468208333333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - }, - "0.01": { - "tp": 322, - "fn": 78, - "accuracy": 0.805 - } - }, - "auroc": 0.9082739583333335 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - }, - "0.01": { - "tp": 341, - "fn": 59, - "accuracy": 0.8525 - } - }, - "auroc": 0.9249260416666668 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9561010416666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 747, - "fn": 53, - "accuracy": 0.93375 - }, - "0.01": { - "tp": 708, - "fn": 92, - "accuracy": 0.885 - } - }, - "auroc": 0.9405135416666667 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9957260416666667 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9916729166666667 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9936994791666667 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.987175 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.972275 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.979725 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9914505208333333 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9819739583333333 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - }, - "0.01": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - } - }, - "auroc": 0.9867122395833332 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9516604166666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9821020833333334 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9668812500000001 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - }, - "0.01": { - "tp": 135, - "fn": 65, - "accuracy": 0.675 - } - }, - "auroc": 0.8588833333333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.95793125 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - } - }, - "auroc": 0.9084072916666668 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - }, - "0.01": { - "tp": 320, - "fn": 80, - "accuracy": 0.8 - } - }, - "auroc": 0.9052718750000001 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9700166666666666 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 762, - "fn": 38, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 707, - "fn": 93, - "accuracy": 0.88375 - } - }, - "auroc": 0.9376442708333333 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9845458333333332 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9639354166666666 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.974240625 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 156, - "fn": 44, - "accuracy": 0.78 - }, - "0.01": { - "tp": 94, - "fn": 106, - "accuracy": 0.47 - } - }, - "auroc": 0.8011135416666667 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9434416666666666 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - }, - "0.01": { - "tp": 280, - "fn": 120, - "accuracy": 0.7 - } - }, - "auroc": 0.8722776041666668 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - }, - "0.01": { - "tp": 292, - "fn": 108, - "accuracy": 0.73 - } - }, - "auroc": 0.8928296875 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9536885416666667 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 753, - "fn": 47, - "accuracy": 0.94125 - }, - "0.01": { - "tp": 668, - "fn": 132, - "accuracy": 0.835 - } - }, - "auroc": 0.9232591145833333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997499999999999 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9984354166666667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990927083333333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9926520833333333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9778770833333333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9852645833333333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9962010416666667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9881562500000001 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - } - }, - "auroc": 0.9921786458333334 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9623989583333334 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9623989583333334 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9428541666666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9428541666666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - } - }, - "auroc": 0.9526265624999999 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - } - }, - "auroc": 0.9526265624999999 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9197968750000001 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9197968750000001 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - }, - "0.01": { - "tp": 137, - "fn": 63, - "accuracy": 0.685 - } - }, - "auroc": 0.8762218749999999 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - }, - "0.01": { - "tp": 137, - "fn": 63, - "accuracy": 0.685 - } - }, - "auroc": 0.8762218749999999 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - }, - "0.01": { - "tp": 301, - "fn": 99, - "accuracy": 0.7525 - } - }, - "auroc": 0.898009375 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - }, - "0.01": { - "tp": 301, - "fn": 99, - "accuracy": 0.7525 - } - }, - "auroc": 0.898009375 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999552083333333 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999552083333333 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995239583333333 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995239583333333 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997395833333333 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997395833333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998427083333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998427083333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9828697916666667 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9828697916666667 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9913562499999999 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9913562499999999 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9337635416666666 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9337635416666666 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.929396875 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.929396875 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - } - }, - "auroc": 0.9315802083333333 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - } - }, - "auroc": 0.9315802083333333 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2171, - "fn": 29, - "accuracy": 0.9868181818181818 - }, - "0.01": { - "tp": 2103, - "fn": 97, - "accuracy": 0.9559090909090909 - } - }, - "auroc": 0.975230965909091 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1191, - "fn": 9, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 1170, - "fn": 30, - "accuracy": 0.975 - } - }, - "auroc": 0.9834852430555555 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3362, - "fn": 38, - "accuracy": 0.9888235294117647 - }, - "0.01": { - "tp": 3273, - "fn": 127, - "accuracy": 0.9626470588235294 - } - }, - "auroc": 0.9781442401960784 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2056, - "fn": 144, - "accuracy": 0.9345454545454546 - }, - "0.01": { - "tp": 1847, - "fn": 353, - "accuracy": 0.8395454545454546 - } - }, - "auroc": 0.9309379734848485 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1175, - "fn": 25, - "accuracy": 0.9791666666666666 - }, - "0.01": { - "tp": 1149, - "fn": 51, - "accuracy": 0.9575 - } - }, - "auroc": 0.96509375 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3231, - "fn": 169, - "accuracy": 0.9502941176470588 - }, - "0.01": { - "tp": 2996, - "fn": 404, - "accuracy": 0.8811764705882353 - } - }, - "auroc": 0.9429929534313726 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 4227, - "fn": 173, - "accuracy": 0.9606818181818182 - }, - "0.01": { - "tp": 3950, - "fn": 450, - "accuracy": 0.8977272727272727 - } - }, - "auroc": 0.9530844696969696 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2366, - "fn": 34, - "accuracy": 0.9858333333333333 - }, - "0.01": { - "tp": 2319, - "fn": 81, - "accuracy": 0.96625 - } - }, - "auroc": 0.9742894965277777 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 6593, - "fn": 207, - "accuracy": 0.9695588235294118 - }, - "0.01": { - "tp": 6269, - "fn": 531, - "accuracy": 0.9219117647058823 - } - }, - "auroc": 0.9605685968137254 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999927083333334 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999666666666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999796875 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999947916666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998040625 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990177083333334 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99999375 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990036458333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994986979166667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9831385416666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9725447916666666 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9778416666666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - } - }, - "auroc": 0.9180187499999999 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9491270833333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9335729166666666 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9505786458333334 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9608359375 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 765, - "fn": 35, - "accuracy": 0.95625 - }, - "0.01": { - "tp": 736, - "fn": 64, - "accuracy": 0.92 - } - }, - "auroc": 0.9557072916666667 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998437499999999 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985895833333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992166666666668 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9972979166666667 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.978053125 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9876755208333333 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985708333333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9883213541666667 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - } - }, - "auroc": 0.99344609375 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9516395833333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9958489583333334 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9737442708333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 156, - "fn": 44, - "accuracy": 0.78 - } - }, - "auroc": 0.9083114583333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9672697916666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - } - }, - "auroc": 0.937790625 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - } - }, - "auroc": 0.9299755208333335 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9815593750000001 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 781, - "fn": 19, - "accuracy": 0.97625 - }, - "0.01": { - "tp": 736, - "fn": 64, - "accuracy": 0.92 - } - }, - "auroc": 0.9557674479166667 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.985653125 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9706656249999999 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.978159375 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - }, - "0.01": { - "tp": 135, - "fn": 65, - "accuracy": 0.675 - } - }, - "auroc": 0.8476145833333333 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9418604166666666 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 320, - "fn": 80, - "accuracy": 0.8 - } - }, - "auroc": 0.8947375 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9166338541666665 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9562630208333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 760, - "fn": 40, - "accuracy": 0.95 - }, - "0.01": { - "tp": 710, - "fn": 90, - "accuracy": 0.8875 - } - }, - "auroc": 0.9364484375000001 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999666666666667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999729166666667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999697916666667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9946125 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9891645833333333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9918885416666667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9972895833333334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.99456875 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9959291666666668 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9845729166666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9845729166666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9744125 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9744125 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9794927083333334 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9794927083333334 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9571145833333334 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9571145833333334 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9254291666666666 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9254291666666666 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.941271875 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.941271875 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999958333333333 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999958333333333 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999677083333334 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999677083333334 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999817708333334 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999817708333334 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999458333333334 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999458333333334 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9947875 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9947875 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9973666666666667 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9973666666666667 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9675927083333333 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9675927083333333 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9557479166666667 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9557479166666667 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9616703125 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9616703125 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2180, - "fn": 20, - "accuracy": 0.990909090909091 - }, - "0.01": { - "tp": 2150, - "fn": 50, - "accuracy": 0.9772727272727273 - } - }, - "auroc": 0.9844960227272728 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1189, - "fn": 11, - "accuracy": 0.9908333333333333 - }, - "0.01": { - "tp": 1180, - "fn": 20, - "accuracy": 0.9833333333333333 - } - }, - "auroc": 0.9895980902777778 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3369, - "fn": 31, - "accuracy": 0.9908823529411764 - }, - "0.01": { - "tp": 3330, - "fn": 70, - "accuracy": 0.9794117647058823 - } - }, - "auroc": 0.9862967524509804 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2123, - "fn": 77, - "accuracy": 0.965 - }, - "0.01": { - "tp": 1999, - "fn": 201, - "accuracy": 0.9086363636363637 - } - }, - "auroc": 0.9560177083333334 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1179, - "fn": 21, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 1157, - "fn": 43, - "accuracy": 0.9641666666666666 - } - }, - "auroc": 0.9705859375 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3302, - "fn": 98, - "accuracy": 0.9711764705882353 - }, - "0.01": { - "tp": 3156, - "fn": 244, - "accuracy": 0.928235294117647 - } - }, - "auroc": 0.9611594362745097 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 4303, - "fn": 97, - "accuracy": 0.9779545454545454 - }, - "0.01": { - "tp": 4149, - "fn": 251, - "accuracy": 0.9429545454545455 - } - }, - "auroc": 0.970256865530303 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2368, - "fn": 32, - "accuracy": 0.9866666666666667 - }, - "0.01": { - "tp": 2337, - "fn": 63, - "accuracy": 0.97375 - } - }, - "auroc": 0.9800920138888889 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 6671, - "fn": 129, - "accuracy": 0.9810294117647059 - }, - "0.01": { - "tp": 6486, - "fn": 314, - "accuracy": 0.9538235294117647 - } - }, - "auroc": 0.9737280943627451 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9968218750000001 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9913739583333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9940979166666668 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998128125 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.972075 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9851015625 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9974749999999999 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9817244791666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - }, - "0.01": { - "tp": 785, - "fn": 15, - "accuracy": 0.98125 - } - }, - "auroc": 0.9895997395833334 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9749322916666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 143, - "fn": 57, - "accuracy": 0.715 - }, - "0.01": { - "tp": 103, - "fn": 97, - "accuracy": 0.515 - } - }, - "auroc": 0.787259375 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - }, - "0.01": { - "tp": 298, - "fn": 102, - "accuracy": 0.745 - } - }, - "auroc": 0.8810958333333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - } - }, - "auroc": 0.9126645833333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - }, - "0.01": { - "tp": 72, - "fn": 128, - "accuracy": 0.36 - } - }, - "auroc": 0.7364479166666668 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 307, - "fn": 93, - "accuracy": 0.7675 - }, - "0.01": { - "tp": 239, - "fn": 161, - "accuracy": 0.5975 - } - }, - "auroc": 0.82455625 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - } - }, - "auroc": 0.9437984375 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 262, - "fn": 138, - "accuracy": 0.655 - }, - "0.01": { - "tp": 175, - "fn": 225, - "accuracy": 0.4375 - } - }, - "auroc": 0.7618536458333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 649, - "fn": 151, - "accuracy": 0.81125 - }, - "0.01": { - "tp": 537, - "fn": 263, - "accuracy": 0.67125 - } - }, - "auroc": 0.8528260416666666 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.98589375 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9538968750000001 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9698953125 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9645250000000001 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.9006875 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - } - }, - "auroc": 0.9326062500000001 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9752093749999999 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 340, - "fn": 60, - "accuracy": 0.85 - } - }, - "auroc": 0.9272921875 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 769, - "fn": 31, - "accuracy": 0.96125 - }, - "0.01": { - "tp": 721, - "fn": 79, - "accuracy": 0.90125 - } - }, - "auroc": 0.95125078125 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9361458333333335 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9443177083333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - } - }, - "auroc": 0.9402317708333334 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.8920874999999999 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - }, - "0.01": { - "tp": 115, - "fn": 85, - "accuracy": 0.575 - } - }, - "auroc": 0.8307302083333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - }, - "0.01": { - "tp": 273, - "fn": 127, - "accuracy": 0.6825 - } - }, - "auroc": 0.8614088541666666 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 336, - "fn": 64, - "accuracy": 0.84 - } - }, - "auroc": 0.9141166666666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - }, - "0.01": { - "tp": 298, - "fn": 102, - "accuracy": 0.745 - } - }, - "auroc": 0.8875239583333334 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 738, - "fn": 62, - "accuracy": 0.9225 - }, - "0.01": { - "tp": 634, - "fn": 166, - "accuracy": 0.7925 - } - }, - "auroc": 0.9008203125 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9738218750000001 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.8899531249999999 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - } - }, - "auroc": 0.9318875 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 138, - "fn": 62, - "accuracy": 0.69 - } - }, - "auroc": 0.86001875 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 137, - "fn": 63, - "accuracy": 0.685 - }, - "0.01": { - "tp": 73, - "fn": 127, - "accuracy": 0.365 - } - }, - "auroc": 0.7650406249999999 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 316, - "fn": 84, - "accuracy": 0.79 - }, - "0.01": { - "tp": 211, - "fn": 189, - "accuracy": 0.5275 - } - }, - "auroc": 0.8125296875000001 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 331, - "fn": 69, - "accuracy": 0.8275 - } - }, - "auroc": 0.9169203125000001 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 321, - "fn": 79, - "accuracy": 0.8025 - }, - "0.01": { - "tp": 231, - "fn": 169, - "accuracy": 0.5775 - } - }, - "auroc": 0.827496875 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 698, - "fn": 102, - "accuracy": 0.8725 - }, - "0.01": { - "tp": 562, - "fn": 238, - "accuracy": 0.7025 - } - }, - "auroc": 0.87220859375 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9961135416666667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9825333333333333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9893234375 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.980203125 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9399302083333333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9600666666666666 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9881583333333334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9612317708333333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 786, - "fn": 14, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 763, - "fn": 37, - "accuracy": 0.95375 - } - }, - "auroc": 0.9746950520833333 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9412749999999999 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9412749999999999 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9193 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9193 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 349, - "fn": 51, - "accuracy": 0.8725 - } - }, - "auroc": 0.9302874999999999 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 349, - "fn": 51, - "accuracy": 0.8725 - } - }, - "auroc": 0.9302874999999999 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - } - }, - "auroc": 0.9240020833333333 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - } - }, - "auroc": 0.9240020833333333 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - } - }, - "auroc": 0.9047802083333334 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - } - }, - "auroc": 0.9047802083333334 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 330, - "fn": 70, - "accuracy": 0.825 - } - }, - "auroc": 0.9143911458333334 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 330, - "fn": 70, - "accuracy": 0.825 - } - }, - "auroc": 0.9143911458333334 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994229166666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994229166666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9966114583333333 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9966114583333333 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9980171874999999 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9980171874999999 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9982927083333334 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9982927083333334 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9877895833333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9877895833333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9930411458333332 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9930411458333332 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9306770833333333 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9306770833333333 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9328916666666667 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9328916666666667 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 340, - "fn": 60, - "accuracy": 0.85 - } - }, - "auroc": 0.9317843750000001 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 340, - "fn": 60, - "accuracy": 0.85 - } - }, - "auroc": 0.9317843750000001 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2154, - "fn": 46, - "accuracy": 0.9790909090909091 - }, - "0.01": { - "tp": 2075, - "fn": 125, - "accuracy": 0.9431818181818182 - } - }, - "auroc": 0.9688544507575758 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1114, - "fn": 86, - "accuracy": 0.9283333333333333 - }, - "0.01": { - "tp": 1017, - "fn": 183, - "accuracy": 0.8475 - } - }, - "auroc": 0.9248890625000001 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3268, - "fn": 132, - "accuracy": 0.9611764705882353 - }, - "0.01": { - "tp": 3092, - "fn": 308, - "accuracy": 0.9094117647058824 - } - }, - "auroc": 0.9533372549019608 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2111, - "fn": 89, - "accuracy": 0.9595454545454546 - }, - "0.01": { - "tp": 1943, - "fn": 257, - "accuracy": 0.8831818181818182 - } - }, - "auroc": 0.9408181818181818 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 997, - "fn": 203, - "accuracy": 0.8308333333333333 - }, - "0.01": { - "tp": 781, - "fn": 419, - "accuracy": 0.6508333333333334 - } - }, - "auroc": 0.8574852430555555 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3108, - "fn": 292, - "accuracy": 0.9141176470588235 - }, - "0.01": { - "tp": 2724, - "fn": 676, - "accuracy": 0.8011764705882353 - } - }, - "auroc": 0.911406556372549 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 4265, - "fn": 135, - "accuracy": 0.9693181818181819 - }, - "0.01": { - "tp": 4018, - "fn": 382, - "accuracy": 0.9131818181818182 - } - }, - "auroc": 0.9548363162878788 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2111, - "fn": 289, - "accuracy": 0.8795833333333334 - }, - "0.01": { - "tp": 1798, - "fn": 602, - "accuracy": 0.7491666666666666 - } - }, - "auroc": 0.8911871527777778 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 6376, - "fn": 424, - "accuracy": 0.9376470588235294 - }, - "0.01": { - "tp": 5816, - "fn": 984, - "accuracy": 0.8552941176470589 - } - }, - "auroc": 0.9323719056372549 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99999375 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998791666666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999364583333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999864583333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9976802083333334 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9988333333333335 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999901041666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987796875000001 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993848958333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.98534375 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9735114583333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9794276041666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - }, - "0.01": { - "tp": 157, - "fn": 43, - "accuracy": 0.785 - } - }, - "auroc": 0.9031697916666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9500541666666668 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 339, - "fn": 61, - "accuracy": 0.8475 - } - }, - "auroc": 0.9266119791666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - } - }, - "auroc": 0.9442567708333334 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9617828125000001 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 755, - "fn": 45, - "accuracy": 0.94375 - }, - "0.01": { - "tp": 724, - "fn": 76, - "accuracy": 0.905 - } - }, - "auroc": 0.9530197916666666 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997770833333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9976364583333335 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987067708333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9970749999999999 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9795 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9882875 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9984260416666667 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9885682291666666 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9934971354166668 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9544104166666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9963760416666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9753932291666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 144, - "fn": 56, - "accuracy": 0.72 - } - }, - "auroc": 0.8925270833333334 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.958703125 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 337, - "fn": 63, - "accuracy": 0.8425 - } - }, - "auroc": 0.9256151041666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 330, - "fn": 70, - "accuracy": 0.825 - } - }, - "auroc": 0.9234687500000001 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9775395833333334 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 782, - "fn": 18, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 723, - "fn": 77, - "accuracy": 0.90375 - } - }, - "auroc": 0.9505041666666667 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9897927083333333 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9586989583333333 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9742458333333333 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - }, - "0.01": { - "tp": 118, - "fn": 82, - "accuracy": 0.59 - } - }, - "auroc": 0.8302385416666667 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.938878125 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - }, - "0.01": { - "tp": 302, - "fn": 98, - "accuracy": 0.755 - } - }, - "auroc": 0.8845583333333333 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - }, - "0.01": { - "tp": 316, - "fn": 84, - "accuracy": 0.79 - } - }, - "auroc": 0.910015625 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - } - }, - "auroc": 0.9487885416666667 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 753, - "fn": 47, - "accuracy": 0.94125 - }, - "0.01": { - "tp": 689, - "fn": 111, - "accuracy": 0.86125 - } - }, - "auroc": 0.9294020833333334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999984375 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999177083333334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999510416666667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9957000000000001 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9890885416666666 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9923942708333333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9978421875000001 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.994503125 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9961726562500001 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9896479166666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9896479166666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9785333333333334 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9785333333333334 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9840906250000001 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9840906250000001 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9562677083333334 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9562677083333334 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.9194427083333333 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.9194427083333333 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 346, - "fn": 54, - "accuracy": 0.865 - } - }, - "auroc": 0.9378552083333332 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 346, - "fn": 54, - "accuracy": 0.865 - } - }, - "auroc": 0.9378552083333332 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999984375 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999984375 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999729166666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999729166666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999786458333333 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999786458333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999916666666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999916666666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9922739583333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9922739583333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9961328125 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9961328125 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.965746875 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.965746875 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9564708333333334 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9564708333333334 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9611088541666667 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9611088541666667 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2183, - "fn": 17, - "accuracy": 0.9922727272727273 - }, - "0.01": { - "tp": 2153, - "fn": 47, - "accuracy": 0.9786363636363636 - } - }, - "auroc": 0.9855400568181818 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1190, - "fn": 10, - "accuracy": 0.9916666666666667 - }, - "0.01": { - "tp": 1176, - "fn": 24, - "accuracy": 0.98 - } - }, - "auroc": 0.9876699652777778 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3373, - "fn": 27, - "accuracy": 0.9920588235294118 - }, - "0.01": { - "tp": 3329, - "fn": 71, - "accuracy": 0.9791176470588235 - } - }, - "auroc": 0.9862917892156863 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2108, - "fn": 92, - "accuracy": 0.9581818181818181 - }, - "0.01": { - "tp": 1953, - "fn": 247, - "accuracy": 0.8877272727272727 - } - }, - "auroc": 0.9513991477272727 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1178, - "fn": 22, - "accuracy": 0.9816666666666667 - }, - "0.01": { - "tp": 1154, - "fn": 46, - "accuracy": 0.9616666666666667 - } - }, - "auroc": 0.9689840277777777 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3286, - "fn": 114, - "accuracy": 0.9664705882352941 - }, - "0.01": { - "tp": 3107, - "fn": 293, - "accuracy": 0.9138235294117647 - } - }, - "auroc": 0.9576055759803921 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 4291, - "fn": 109, - "accuracy": 0.9752272727272727 - }, - "0.01": { - "tp": 4106, - "fn": 294, - "accuracy": 0.9331818181818182 - } - }, - "auroc": 0.9684696022727273 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2368, - "fn": 32, - "accuracy": 0.9866666666666667 - }, - "0.01": { - "tp": 2330, - "fn": 70, - "accuracy": 0.9708333333333333 - } - }, - "auroc": 0.9783269965277777 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 6659, - "fn": 141, - "accuracy": 0.9792647058823529 - }, - "0.01": { - "tp": 6436, - "fn": 364, - "accuracy": 0.9464705882352941 - } - }, - "auroc": 0.9719486825980392 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999843749999999 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998541666666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999192708333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999802083333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9967541666666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9983671875000001 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9983041666666668 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9991432291666666 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9818312499999999 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9735958333333334 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9777135416666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - }, - "0.01": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - } - }, - "auroc": 0.8561875 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9492052083333332 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 346, - "fn": 54, - "accuracy": 0.865 - }, - "0.01": { - "tp": 317, - "fn": 83, - "accuracy": 0.7925 - } - }, - "auroc": 0.9026963541666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9190093750000001 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9614005208333334 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 738, - "fn": 62, - "accuracy": 0.9225 - }, - "0.01": { - "tp": 703, - "fn": 97, - "accuracy": 0.87875 - } - }, - "auroc": 0.9402049479166666 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999784375 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980239583333335 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989041666666667 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9970322916666667 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9826937499999999 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9898630208333333 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9984083333333333 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9903588541666667 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - } - }, - "auroc": 0.9943835937500001 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9519604166666666 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9919072916666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9719338541666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 115, - "fn": 85, - "accuracy": 0.575 - } - }, - "auroc": 0.8506145833333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9456739583333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 304, - "fn": 96, - "accuracy": 0.76 - } - }, - "auroc": 0.8981442708333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - }, - "0.01": { - "tp": 301, - "fn": 99, - "accuracy": 0.7525 - } - }, - "auroc": 0.9012875 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.968790625 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 766, - "fn": 34, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 688, - "fn": 112, - "accuracy": 0.86 - } - }, - "auroc": 0.9350390625 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9850749999999999 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9397979166666668 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9624364583333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 129, - "fn": 71, - "accuracy": 0.645 - }, - "0.01": { - "tp": 86, - "fn": 114, - "accuracy": 0.43 - } - }, - "auroc": 0.7649208333333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9221364583333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 323, - "fn": 77, - "accuracy": 0.8075 - }, - "0.01": { - "tp": 264, - "fn": 136, - "accuracy": 0.66 - } - }, - "auroc": 0.8435286458333333 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 329, - "fn": 71, - "accuracy": 0.8225 - }, - "0.01": { - "tp": 283, - "fn": 117, - "accuracy": 0.7075 - } - }, - "auroc": 0.8749979166666667 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.9309671875000001 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 718, - "fn": 82, - "accuracy": 0.8975 - }, - "0.01": { - "tp": 643, - "fn": 157, - "accuracy": 0.80375 - } - }, - "auroc": 0.9029825520833333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999666666666667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999333333333333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9948593750000001 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9893125 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9920859375000002 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9974130208333334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9946062499999999 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9960096354166668 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9776010416666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9776010416666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.966853125 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.966853125 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9722270833333333 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9722270833333333 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - } - }, - "auroc": 0.9234739583333333 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - } - }, - "auroc": 0.9234739583333333 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 142, - "fn": 58, - "accuracy": 0.71 - } - }, - "auroc": 0.87948125 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 142, - "fn": 58, - "accuracy": 0.71 - } - }, - "auroc": 0.87948125 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 312, - "fn": 88, - "accuracy": 0.78 - } - }, - "auroc": 0.9014776041666668 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 312, - "fn": 88, - "accuracy": 0.78 - } - }, - "auroc": 0.9014776041666668 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999614583333334 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999614583333334 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999971875 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999971875 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999940625 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999940625 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9894385416666667 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9894385416666667 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9946895833333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9946895833333333 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.9459968750000001 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.9459968750000001 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9416052083333333 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9416052083333333 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9438010416666667 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9438010416666667 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2173, - "fn": 27, - "accuracy": 0.9877272727272727 - }, - "0.01": { - "tp": 2121, - "fn": 79, - "accuracy": 0.9640909090909091 - } - }, - "auroc": 0.978690625 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1187, - "fn": 13, - "accuracy": 0.9891666666666666 - }, - "0.01": { - "tp": 1168, - "fn": 32, - "accuracy": 0.9733333333333334 - } - }, - "auroc": 0.9838465277777777 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3360, - "fn": 40, - "accuracy": 0.9882352941176471 - }, - "0.01": { - "tp": 3289, - "fn": 111, - "accuracy": 0.9673529411764706 - } - }, - "auroc": 0.9805103553921568 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2027, - "fn": 173, - "accuracy": 0.9213636363636364 - }, - "0.01": { - "tp": 1842, - "fn": 358, - "accuracy": 0.8372727272727273 - } - }, - "auroc": 0.9309940340909091 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1176, - "fn": 24, - "accuracy": 0.98 - }, - "0.01": { - "tp": 1143, - "fn": 57, - "accuracy": 0.9525 - } - }, - "auroc": 0.9642960069444444 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3203, - "fn": 197, - "accuracy": 0.9420588235294117 - }, - "0.01": { - "tp": 2985, - "fn": 415, - "accuracy": 0.8779411764705882 - } - }, - "auroc": 0.9427476715686274 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 4200, - "fn": 200, - "accuracy": 0.9545454545454546 - }, - "0.01": { - "tp": 3963, - "fn": 437, - "accuracy": 0.9006818181818181 - } - }, - "auroc": 0.9548423295454546 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2363, - "fn": 37, - "accuracy": 0.9845833333333334 - }, - "0.01": { - "tp": 2311, - "fn": 89, - "accuracy": 0.9629166666666666 - } - }, - "auroc": 0.9740712673611112 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 6563, - "fn": 237, - "accuracy": 0.9651470588235294 - }, - "0.01": { - "tp": 6274, - "fn": 526, - "accuracy": 0.9226470588235294 - } - }, - "auroc": 0.9616290134803922 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9052354166666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.88493125 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.8950833333333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9096541666666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.8984677083333334 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9040609374999999 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9074447916666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.8916994791666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 774, - "fn": 26, - "accuracy": 0.9675 - } - }, - "auroc": 0.8995721354166667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9947520833333334 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9620708333333334 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9784114583333334 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9561875 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9610510416666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9586192708333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9754697916666668 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9615609375 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 786, - "fn": 14, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 780, - "fn": 20, - "accuracy": 0.975 - } - }, - "auroc": 0.9685153645833333 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9282604166666666 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.92715625 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9277083333333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9205333333333333 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.935346875 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9279401041666666 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.924396875 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9312515625 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.92782421875 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9628708333333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.968659375 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9657651041666666 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9713635416666666 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9675552083333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.969459375 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9671171875 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9681072916666666 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 789, - "fn": 11, - "accuracy": 0.98625 - }, - "0.01": { - "tp": 780, - "fn": 20, - "accuracy": 0.975 - } - }, - "auroc": 0.9676122395833333 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9952802083333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9628395833333333 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9790598958333333 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.963921875 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9612208333333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9625713541666667 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9796010416666667 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9620302083333332 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.970815625 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9433906249999999 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.907234375 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9253125 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9311854166666667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9227562500000001 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9269708333333334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9372880208333334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9149953125 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 788, - "fn": 12, - "accuracy": 0.985 - } - }, - "auroc": 0.9261416666666666 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9748645833333334 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9748645833333334 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9669187499999999 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9669187499999999 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9708916666666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9708916666666667 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9607500000000001 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9607500000000001 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9511218749999999 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9511218749999999 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9559359375 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9559359375 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9010989583333333 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9010989583333333 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.8972947916666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.8972947916666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.899196875 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.899196875 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9096072916666667 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9096072916666667 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.923059375 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.923059375 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9163333333333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9163333333333333 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9524885416666666 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9524885416666666 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9415354166666667 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9415354166666667 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9470119791666667 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9470119791666667 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2191, - "fn": 9, - "accuracy": 0.9959090909090909 - }, - "0.01": { - "tp": 2166, - "fn": 34, - "accuracy": 0.9845454545454545 - } - }, - "auroc": 0.9480544507575759 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1195, - "fn": 5, - "accuracy": 0.9958333333333333 - }, - "0.01": { - "tp": 1180, - "fn": 20, - "accuracy": 0.9833333333333333 - } - }, - "auroc": 0.9354819444444444 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3386, - "fn": 14, - "accuracy": 0.9958823529411764 - }, - "0.01": { - "tp": 3346, - "fn": 54, - "accuracy": 0.9841176470588235 - } - }, - "auroc": 0.9436170955882353 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2196, - "fn": 4, - "accuracy": 0.9981818181818182 - }, - "0.01": { - "tp": 2182, - "fn": 18, - "accuracy": 0.9918181818181818 - } - }, - "auroc": 0.9393432765151515 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1188, - "fn": 12, - "accuracy": 0.99 - }, - "0.01": { - "tp": 1180, - "fn": 20, - "accuracy": 0.9833333333333333 - } - }, - "auroc": 0.9410663194444444 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3384, - "fn": 16, - "accuracy": 0.9952941176470588 - }, - "0.01": { - "tp": 3362, - "fn": 38, - "accuracy": 0.9888235294117647 - } - }, - "auroc": 0.9399514093137256 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 4387, - "fn": 13, - "accuracy": 0.9970454545454546 - }, - "0.01": { - "tp": 4348, - "fn": 52, - "accuracy": 0.9881818181818182 - } - }, - "auroc": 0.9436988636363637 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2383, - "fn": 17, - "accuracy": 0.9929166666666667 - }, - "0.01": { - "tp": 2360, - "fn": 40, - "accuracy": 0.9833333333333333 - } - }, - "auroc": 0.9382741319444445 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 6770, - "fn": 30, - "accuracy": 0.9955882352941177 - }, - "0.01": { - "tp": 6708, - "fn": 92, - "accuracy": 0.9864705882352941 - } - }, - "auroc": 0.9417842524509804 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9909270833333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9880510416666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9894890624999999 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9903364583333334 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9835437499999999 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9869401041666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9906317708333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9857973958333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9882145833333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9386427083333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.97351875 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9560807291666668 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 157, - "fn": 43, - "accuracy": 0.785 - } - }, - "auroc": 0.8771 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9490958333333332 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 340, - "fn": 60, - "accuracy": 0.85 - } - }, - "auroc": 0.9130979166666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9078713541666666 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9613072916666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 772, - "fn": 28, - "accuracy": 0.965 - }, - "0.01": { - "tp": 723, - "fn": 77, - "accuracy": 0.90375 - } - }, - "auroc": 0.9345893229166666 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9890239583333333 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9949875 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9920057291666666 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9841333333333333 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.978690625 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9814119791666668 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9865786458333333 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9868390625000001 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9867088541666669 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9219739583333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9682395833333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9451067708333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 160, - "fn": 40, - "accuracy": 0.8 - } - }, - "auroc": 0.8571760416666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.965875 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9115255208333334 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 340, - "fn": 60, - "accuracy": 0.85 - } - }, - "auroc": 0.889575 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9670572916666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 787, - "fn": 13, - "accuracy": 0.98375 - }, - "0.01": { - "tp": 734, - "fn": 66, - "accuracy": 0.9175 - } - }, - "auroc": 0.9283161458333333 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9438583333333332 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9406833333333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9422708333333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 97, - "fn": 103, - "accuracy": 0.485 - } - }, - "auroc": 0.8203229166666668 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9379406250000001 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 284, - "fn": 116, - "accuracy": 0.71 - } - }, - "auroc": 0.8791317708333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 291, - "fn": 109, - "accuracy": 0.7275 - } - }, - "auroc": 0.882090625 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9393119791666666 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 784, - "fn": 16, - "accuracy": 0.98 - }, - "0.01": { - "tp": 669, - "fn": 131, - "accuracy": 0.83625 - } - }, - "auroc": 0.9107013020833334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.991875 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.992809375 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9923421875 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9782114583333333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9821395833333333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9801755208333333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9850432291666668 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9874744791666668 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9862588541666666 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9235447916666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9235447916666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9108760416666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9108760416666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9172104166666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9172104166666667 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.8673927083333333 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.8673927083333333 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 135, - "fn": 65, - "accuracy": 0.675 - } - }, - "auroc": 0.8392072916666667 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 135, - "fn": 65, - "accuracy": 0.675 - } - }, - "auroc": 0.8392072916666667 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 307, - "fn": 93, - "accuracy": 0.7675 - } - }, - "auroc": 0.8533 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 307, - "fn": 93, - "accuracy": 0.7675 - } - }, - "auroc": 0.8533 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9948166666666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9948166666666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9933177083333333 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9933177083333333 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9940671875 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9940671875 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9880343749999999 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9880343749999999 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9406260416666667 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9406260416666667 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9643302083333334 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9643302083333334 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.917178125 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.917178125 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.8970166666666667 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.8970166666666667 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 333, - "fn": 67, - "accuracy": 0.8325 - } - }, - "auroc": 0.9070973958333334 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 333, - "fn": 67, - "accuracy": 0.8325 - } - }, - "auroc": 0.9070973958333334 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2187, - "fn": 13, - "accuracy": 0.9940909090909091 - }, - "0.01": { - "tp": 2104, - "fn": 96, - "accuracy": 0.9563636363636364 - } - }, - "auroc": 0.9515697916666666 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1192, - "fn": 8, - "accuracy": 0.9933333333333333 - }, - "0.01": { - "tp": 1177, - "fn": 23, - "accuracy": 0.9808333333333333 - } - }, - "auroc": 0.9763815972222221 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3379, - "fn": 21, - "accuracy": 0.9938235294117647 - }, - "0.01": { - "tp": 3281, - "fn": 119, - "accuracy": 0.965 - } - }, - "auroc": 0.960326899509804 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2162, - "fn": 38, - "accuracy": 0.9827272727272728 - }, - "0.01": { - "tp": 1892, - "fn": 308, - "accuracy": 0.86 - } - }, - "auroc": 0.9171203598484847 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1180, - "fn": 20, - "accuracy": 0.9833333333333333 - }, - "0.01": { - "tp": 1161, - "fn": 39, - "accuracy": 0.9675 - } - }, - "auroc": 0.9662142361111112 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3342, - "fn": 58, - "accuracy": 0.9829411764705882 - }, - "0.01": { - "tp": 3053, - "fn": 347, - "accuracy": 0.8979411764705882 - } - }, - "auroc": 0.9344476102941175 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 4349, - "fn": 51, - "accuracy": 0.9884090909090909 - }, - "0.01": { - "tp": 3996, - "fn": 404, - "accuracy": 0.9081818181818182 - } - }, - "auroc": 0.9343450757575757 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2372, - "fn": 28, - "accuracy": 0.9883333333333333 - }, - "0.01": { - "tp": 2338, - "fn": 62, - "accuracy": 0.9741666666666666 - } - }, - "auroc": 0.9712979166666666 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 6721, - "fn": 79, - "accuracy": 0.9883823529411765 - }, - "0.01": { - "tp": 6334, - "fn": 466, - "accuracy": 0.9314705882352942 - } - }, - "auroc": 0.9473872549019607 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999947916666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999896875 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999458333333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999791666666668 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9960447916666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980119791666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999869791666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9979708333333334 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989789062500001 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9828770833333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9732083333333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9780427083333334 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - }, - "0.01": { - "tp": 142, - "fn": 58, - "accuracy": 0.71 - } - }, - "auroc": 0.878865625 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9487770833333333 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - }, - "0.01": { - "tp": 324, - "fn": 76, - "accuracy": 0.81 - } - }, - "auroc": 0.9138213541666668 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - }, - "0.01": { - "tp": 340, - "fn": 60, - "accuracy": 0.85 - } - }, - "auroc": 0.9308713541666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9609927083333334 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 752, - "fn": 48, - "accuracy": 0.94 - }, - "0.01": { - "tp": 710, - "fn": 90, - "accuracy": 0.8875 - } - }, - "auroc": 0.9459320312500001 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997760416666666 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9977093750000001 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987427083333333 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9972625 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9793333333333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9882979166666668 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985192708333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9885213541666668 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9935203125000001 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9522354166666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9929697916666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9726026041666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 141, - "fn": 59, - "accuracy": 0.705 - } - }, - "auroc": 0.8828125 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9602187500000001 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 336, - "fn": 64, - "accuracy": 0.84 - } - }, - "auroc": 0.921515625 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 327, - "fn": 73, - "accuracy": 0.8175 - } - }, - "auroc": 0.9175239583333334 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9765942708333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 773, - "fn": 27, - "accuracy": 0.96625 - }, - "0.01": { - "tp": 720, - "fn": 80, - "accuracy": 0.9 - } - }, - "auroc": 0.9470591145833334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9855437499999999 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.96245625 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.974 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - }, - "0.01": { - "tp": 110, - "fn": 90, - "accuracy": 0.55 - } - }, - "auroc": 0.8095791666666666 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9317104166666667 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 340, - "fn": 60, - "accuracy": 0.85 - }, - "0.01": { - "tp": 289, - "fn": 111, - "accuracy": 0.7225 - } - }, - "auroc": 0.8706447916666666 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 347, - "fn": 53, - "accuracy": 0.8675 - }, - "0.01": { - "tp": 307, - "fn": 93, - "accuracy": 0.7675 - } - }, - "auroc": 0.8975614583333333 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9470833333333333 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 737, - "fn": 63, - "accuracy": 0.92125 - }, - "0.01": { - "tp": 675, - "fn": 125, - "accuracy": 0.84375 - } - }, - "auroc": 0.9223223958333334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999520833333333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998854166666666 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99991875 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9950947916666667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9889437500000001 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9920192708333334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9975234374999999 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9944145833333334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9959690104166667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.982796875 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.982796875 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9711010416666668 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9711010416666668 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9769489583333334 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9769489583333334 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9444999999999999 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9444999999999999 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.9147000000000001 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.9147000000000001 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - } - }, - "auroc": 0.9296 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - } - }, - "auroc": 0.9296 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999854166666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999854166666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999572916666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999572916666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999713541666666 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999713541666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999885416666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999885416666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9907406250000002 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9907406250000002 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9953645833333333 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9953645833333333 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9598489583333334 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9598489583333334 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9488489583333333 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9488489583333333 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9543489583333333 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9543489583333333 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2180, - "fn": 20, - "accuracy": 0.990909090909091 - }, - "0.01": { - "tp": 2140, - "fn": 60, - "accuracy": 0.9727272727272728 - } - }, - "auroc": 0.9824999053030303 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1189, - "fn": 11, - "accuracy": 0.9908333333333333 - }, - "0.01": { - "tp": 1175, - "fn": 25, - "accuracy": 0.9791666666666666 - } - }, - "auroc": 0.9876876736111111 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3369, - "fn": 31, - "accuracy": 0.9908823529411764 - }, - "0.01": { - "tp": 3315, - "fn": 85, - "accuracy": 0.975 - } - }, - "auroc": 0.9843308823529411 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2081, - "fn": 119, - "accuracy": 0.9459090909090909 - }, - "0.01": { - "tp": 1918, - "fn": 282, - "accuracy": 0.8718181818181818 - } - }, - "auroc": 0.9444492424242424 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1176, - "fn": 24, - "accuracy": 0.98 - }, - "0.01": { - "tp": 1152, - "fn": 48, - "accuracy": 0.96 - } - }, - "auroc": 0.9675046875000001 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3257, - "fn": 143, - "accuracy": 0.9579411764705882 - }, - "0.01": { - "tp": 3070, - "fn": 330, - "accuracy": 0.9029411764705882 - } - }, - "auroc": 0.9525864583333333 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 4261, - "fn": 139, - "accuracy": 0.9684090909090909 - }, - "0.01": { - "tp": 4058, - "fn": 342, - "accuracy": 0.9222727272727272 - } - }, - "auroc": 0.9634745738636363 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2365, - "fn": 35, - "accuracy": 0.9854166666666667 - }, - "0.01": { - "tp": 2327, - "fn": 73, - "accuracy": 0.9695833333333334 - } - }, - "auroc": 0.9775961805555555 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 6626, - "fn": 174, - "accuracy": 0.9744117647058823 - }, - "0.01": { - "tp": 6385, - "fn": 415, - "accuracy": 0.9389705882352941 - } - }, - "auroc": 0.9684586703431373 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998489583333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999156250000001 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999791666666666 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9967677083333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9983734375 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999807291666667 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9983083333333334 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.99914453125 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9817760416666668 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.97351875 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9776473958333334 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - }, - "0.01": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - } - }, - "auroc": 0.8585822916666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9490958333333332 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 349, - "fn": 51, - "accuracy": 0.8725 - }, - "0.01": { - "tp": 317, - "fn": 83, - "accuracy": 0.7925 - } - }, - "auroc": 0.9038390625 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9201791666666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9613072916666667 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 741, - "fn": 59, - "accuracy": 0.92625 - }, - "0.01": { - "tp": 703, - "fn": 97, - "accuracy": 0.87875 - } - }, - "auroc": 0.9407432291666668 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997510416666666 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9979958333333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9988734375 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9970187500000001 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9781083333333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9875635416666667 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983848958333335 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9880520833333334 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - } - }, - "auroc": 0.9932184895833333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.95169375 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9918156250000001 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9717546875 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 116, - "fn": 84, - "accuracy": 0.58 - } - }, - "auroc": 0.8512177083333333 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9464479166666666 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 307, - "fn": 93, - "accuracy": 0.7675 - } - }, - "auroc": 0.8988328125 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 302, - "fn": 98, - "accuracy": 0.755 - } - }, - "auroc": 0.9014557291666667 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9691317708333334 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - }, - "0.01": { - "tp": 691, - "fn": 109, - "accuracy": 0.86375 - } - }, - "auroc": 0.93529375 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.98485 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9401322916666667 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9624911458333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 129, - "fn": 71, - "accuracy": 0.645 - }, - "0.01": { - "tp": 86, - "fn": 114, - "accuracy": 0.43 - } - }, - "auroc": 0.7668166666666666 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9213927083333334 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 321, - "fn": 79, - "accuracy": 0.8025 - }, - "0.01": { - "tp": 265, - "fn": 135, - "accuracy": 0.6625 - } - }, - "auroc": 0.8441046875 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 329, - "fn": 71, - "accuracy": 0.8225 - }, - "0.01": { - "tp": 283, - "fn": 117, - "accuracy": 0.7075 - } - }, - "auroc": 0.8758333333333335 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - } - }, - "auroc": 0.9307625 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 716, - "fn": 84, - "accuracy": 0.895 - }, - "0.01": { - "tp": 646, - "fn": 154, - "accuracy": 0.8075 - } - }, - "auroc": 0.9032979166666666 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99995625 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999884375 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999203125 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9948614583333335 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9879322916666666 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9913968750000001 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9974088541666667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9939083333333334 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9956585937500001 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.977653125 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.977653125 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9677802083333333 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9677802083333333 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9727166666666667 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9727166666666667 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - } - }, - "auroc": 0.9237114583333332 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - } - }, - "auroc": 0.9237114583333332 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 139, - "fn": 61, - "accuracy": 0.695 - } - }, - "auroc": 0.8780041666666667 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 139, - "fn": 61, - "accuracy": 0.695 - } - }, - "auroc": 0.8780041666666667 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 309, - "fn": 91, - "accuracy": 0.7725 - } - }, - "auroc": 0.9008578125 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 309, - "fn": 91, - "accuracy": 0.7725 - } - }, - "auroc": 0.9008578125 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99998125 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99998125 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999604166666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999604166666667 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999708333333334 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999708333333334 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999385416666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999385416666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9894635416666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9894635416666666 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9947010416666667 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9947010416666667 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.945846875 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.945846875 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9385531250000001 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9385531250000001 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9422 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9422 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2174, - "fn": 26, - "accuracy": 0.9881818181818182 - }, - "0.01": { - "tp": 2121, - "fn": 79, - "accuracy": 0.9640909090909091 - } - }, - "auroc": 0.9786491477272727 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1187, - "fn": 13, - "accuracy": 0.9891666666666666 - }, - "0.01": { - "tp": 1170, - "fn": 30, - "accuracy": 0.975 - } - }, - "auroc": 0.9838659722222223 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3361, - "fn": 39, - "accuracy": 0.9885294117647059 - }, - "0.01": { - "tp": 3291, - "fn": 109, - "accuracy": 0.9679411764705882 - } - }, - "auroc": 0.9804903799019608 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2034, - "fn": 166, - "accuracy": 0.9245454545454546 - }, - "0.01": { - "tp": 1840, - "fn": 360, - "accuracy": 0.8363636363636363 - } - }, - "auroc": 0.9311125 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1174, - "fn": 26, - "accuracy": 0.9783333333333334 - }, - "0.01": { - "tp": 1146, - "fn": 54, - "accuracy": 0.955 - } - }, - "auroc": 0.9632907986111111 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3208, - "fn": 192, - "accuracy": 0.9435294117647058 - }, - "0.01": { - "tp": 2986, - "fn": 414, - "accuracy": 0.8782352941176471 - } - }, - "auroc": 0.9424695465686275 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 4208, - "fn": 192, - "accuracy": 0.9563636363636364 - }, - "0.01": { - "tp": 3961, - "fn": 439, - "accuracy": 0.9002272727272728 - } - }, - "auroc": 0.9548808238636364 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2361, - "fn": 39, - "accuracy": 0.98375 - }, - "0.01": { - "tp": 2316, - "fn": 84, - "accuracy": 0.965 - } - }, - "auroc": 0.9735783854166666 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 6569, - "fn": 231, - "accuracy": 0.9660294117647059 - }, - "0.01": { - "tp": 6277, - "fn": 523, - "accuracy": 0.9230882352941177 - } - }, - "auroc": 0.961479963235294 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2392, - "fn": 8, - "accuracy": 0.9966666666666667 - } - }, - "auroc": 0.9910708333333333 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2392, - "fn": 8, - "accuracy": 0.9966666666666667 - } - }, - "auroc": 0.9885203124999999 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - }, - "0.01": { - "tp": 4784, - "fn": 16, - "accuracy": 0.9966666666666667 - } - }, - "auroc": 0.9897955729166666 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2393, - "fn": 7, - "accuracy": 0.9970833333333333 - } - }, - "auroc": 0.9914467013888889 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2397, - "fn": 3, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 2377, - "fn": 23, - "accuracy": 0.9904166666666666 - } - }, - "auroc": 0.985163888888889 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4797, - "fn": 3, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 4770, - "fn": 30, - "accuracy": 0.99375 - } - }, - "auroc": 0.988305295138889 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4785, - "fn": 15, - "accuracy": 0.996875 - } - }, - "auroc": 0.9912587673611111 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4796, - "fn": 4, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 4769, - "fn": 31, - "accuracy": 0.9935416666666667 - } - }, - "auroc": 0.9868421006944444 - }, - { - "domain": "news", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9596, - "fn": 4, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 9554, - "fn": 46, - "accuracy": 0.9952083333333334 - } - }, - "auroc": 0.9890504340277778 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 2376, - "fn": 24, - "accuracy": 0.99 - } - }, - "auroc": 0.980348263888889 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2259, - "fn": 141, - "accuracy": 0.94125 - }, - "0.01": { - "tp": 2175, - "fn": 225, - "accuracy": 0.90625 - } - }, - "auroc": 0.9561378472222222 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4657, - "fn": 143, - "accuracy": 0.9702083333333333 - }, - "0.01": { - "tp": 4551, - "fn": 249, - "accuracy": 0.948125 - } - }, - "auroc": 0.9682430555555555 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2120, - "fn": 280, - "accuracy": 0.8833333333333333 - }, - "0.01": { - "tp": 1857, - "fn": 543, - "accuracy": 0.77375 - } - }, - "auroc": 0.8915455729166666 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2189, - "fn": 211, - "accuracy": 0.9120833333333334 - }, - "0.01": { - "tp": 2087, - "fn": 313, - "accuracy": 0.8695833333333334 - } - }, - "auroc": 0.9319856770833332 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4309, - "fn": 491, - "accuracy": 0.8977083333333333 - }, - "0.01": { - "tp": 3944, - "fn": 856, - "accuracy": 0.8216666666666667 - } - }, - "auroc": 0.9117656250000001 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4518, - "fn": 282, - "accuracy": 0.94125 - }, - "0.01": { - "tp": 4233, - "fn": 567, - "accuracy": 0.881875 - } - }, - "auroc": 0.9359469184027778 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4448, - "fn": 352, - "accuracy": 0.9266666666666666 - }, - "0.01": { - "tp": 4262, - "fn": 538, - "accuracy": 0.8879166666666667 - } - }, - "auroc": 0.9440617621527777 - }, - { - "domain": "news", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 8966, - "fn": 634, - "accuracy": 0.9339583333333333 - }, - "0.01": { - "tp": 8495, - "fn": 1105, - "accuracy": 0.8848958333333333 - } - }, - "auroc": 0.9400043402777777 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9914184895833333 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2392, - "fn": 8, - "accuracy": 0.9966666666666667 - }, - "0.01": { - "tp": 2382, - "fn": 18, - "accuracy": 0.9925 - } - }, - "auroc": 0.9875769097222222 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4792, - "fn": 8, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 4778, - "fn": 22, - "accuracy": 0.9954166666666666 - } - }, - "auroc": 0.9894976996527777 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2391, - "fn": 9, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 2380, - "fn": 20, - "accuracy": 0.9916666666666667 - } - }, - "auroc": 0.9860969618055555 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2364, - "fn": 36, - "accuracy": 0.985 - }, - "0.01": { - "tp": 2331, - "fn": 69, - "accuracy": 0.97125 - } - }, - "auroc": 0.9681845486111111 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4755, - "fn": 45, - "accuracy": 0.990625 - }, - "0.01": { - "tp": 4711, - "fn": 89, - "accuracy": 0.9814583333333333 - } - }, - "auroc": 0.9771407552083333 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4791, - "fn": 9, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 4776, - "fn": 24, - "accuracy": 0.995 - } - }, - "auroc": 0.9887577256944444 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4756, - "fn": 44, - "accuracy": 0.9908333333333333 - }, - "0.01": { - "tp": 4713, - "fn": 87, - "accuracy": 0.981875 - } - }, - "auroc": 0.9778807291666667 - }, - { - "domain": "news", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9547, - "fn": 53, - "accuracy": 0.9944791666666667 - }, - "0.01": { - "tp": 9489, - "fn": 111, - "accuracy": 0.9884375 - } - }, - "auroc": 0.9833192274305556 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2292, - "fn": 108, - "accuracy": 0.955 - }, - "0.01": { - "tp": 2223, - "fn": 177, - "accuracy": 0.92625 - } - }, - "auroc": 0.9518630208333334 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 2363, - "fn": 37, - "accuracy": 0.9845833333333334 - } - }, - "auroc": 0.9844668402777778 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4688, - "fn": 112, - "accuracy": 0.9766666666666667 - }, - "0.01": { - "tp": 4586, - "fn": 214, - "accuracy": 0.9554166666666667 - } - }, - "auroc": 0.9681649305555555 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2250, - "fn": 150, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 1749, - "fn": 651, - "accuracy": 0.72875 - } - }, - "auroc": 0.8850900173611111 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2339, - "fn": 61, - "accuracy": 0.9745833333333334 - }, - "0.01": { - "tp": 2244, - "fn": 156, - "accuracy": 0.935 - } - }, - "auroc": 0.9477903645833332 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4589, - "fn": 211, - "accuracy": 0.9560416666666667 - }, - "0.01": { - "tp": 3993, - "fn": 807, - "accuracy": 0.831875 - } - }, - "auroc": 0.9164401909722222 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4542, - "fn": 258, - "accuracy": 0.94625 - }, - "0.01": { - "tp": 3972, - "fn": 828, - "accuracy": 0.8275 - } - }, - "auroc": 0.9184765190972223 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4735, - "fn": 65, - "accuracy": 0.9864583333333333 - }, - "0.01": { - "tp": 4607, - "fn": 193, - "accuracy": 0.9597916666666667 - } - }, - "auroc": 0.9661286024305555 - }, - { - "domain": "news", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9277, - "fn": 323, - "accuracy": 0.9663541666666666 - }, - "0.01": { - "tp": 8579, - "fn": 1021, - "accuracy": 0.8936458333333334 - } - }, - "auroc": 0.9423025607638889 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 2361, - "fn": 39, - "accuracy": 0.98375 - } - }, - "auroc": 0.98271015625 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2355, - "fn": 45, - "accuracy": 0.98125 - }, - "0.01": { - "tp": 2244, - "fn": 156, - "accuracy": 0.935 - } - }, - "auroc": 0.9499519097222222 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4753, - "fn": 47, - "accuracy": 0.9902083333333334 - }, - "0.01": { - "tp": 4605, - "fn": 195, - "accuracy": 0.959375 - } - }, - "auroc": 0.9663310329861111 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 1908, - "fn": 492, - "accuracy": 0.795 - }, - "0.01": { - "tp": 1421, - "fn": 979, - "accuracy": 0.5920833333333333 - } - }, - "auroc": 0.8261980034722222 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2286, - "fn": 114, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 2103, - "fn": 297, - "accuracy": 0.87625 - } - }, - "auroc": 0.92193515625 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4194, - "fn": 606, - "accuracy": 0.87375 - }, - "0.01": { - "tp": 3524, - "fn": 1276, - "accuracy": 0.7341666666666666 - } - }, - "auroc": 0.8740665798611111 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4306, - "fn": 494, - "accuracy": 0.8970833333333333 - }, - "0.01": { - "tp": 3782, - "fn": 1018, - "accuracy": 0.7879166666666667 - } - }, - "auroc": 0.9044540798611111 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4641, - "fn": 159, - "accuracy": 0.966875 - }, - "0.01": { - "tp": 4347, - "fn": 453, - "accuracy": 0.905625 - } - }, - "auroc": 0.9359435329861111 - }, - { - "domain": "news", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 8947, - "fn": 653, - "accuracy": 0.9319791666666667 - }, - "0.01": { - "tp": 8129, - "fn": 1471, - "accuracy": 0.8467708333333334 - } - }, - "auroc": 0.9201988064236112 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9942321180555556 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2386, - "fn": 14, - "accuracy": 0.9941666666666666 - } - }, - "auroc": 0.9899757812500001 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4785, - "fn": 15, - "accuracy": 0.996875 - } - }, - "auroc": 0.9921039496527777 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2386, - "fn": 14, - "accuracy": 0.9941666666666666 - }, - "0.01": { - "tp": 2379, - "fn": 21, - "accuracy": 0.99125 - } - }, - "auroc": 0.9869372395833333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2377, - "fn": 23, - "accuracy": 0.9904166666666666 - }, - "0.01": { - "tp": 2345, - "fn": 55, - "accuracy": 0.9770833333333333 - } - }, - "auroc": 0.9778903645833332 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4763, - "fn": 37, - "accuracy": 0.9922916666666667 - }, - "0.01": { - "tp": 4724, - "fn": 76, - "accuracy": 0.9841666666666666 - } - }, - "auroc": 0.9824138020833333 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4786, - "fn": 14, - "accuracy": 0.9970833333333333 - }, - "0.01": { - "tp": 4778, - "fn": 22, - "accuracy": 0.9954166666666666 - } - }, - "auroc": 0.9905846788194446 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4777, - "fn": 23, - "accuracy": 0.9952083333333334 - }, - "0.01": { - "tp": 4731, - "fn": 69, - "accuracy": 0.985625 - } - }, - "auroc": 0.9839330729166667 - }, - { - "domain": "news", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9563, - "fn": 37, - "accuracy": 0.9961458333333333 - }, - "0.01": { - "tp": 9509, - "fn": 91, - "accuracy": 0.9905208333333333 - } - }, - "auroc": 0.9872588758680555 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2383, - "fn": 17, - "accuracy": 0.9929166666666667 - }, - "0.01": { - "tp": 2328, - "fn": 72, - "accuracy": 0.97 - } - }, - "auroc": 0.971588888888889 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2383, - "fn": 17, - "accuracy": 0.9929166666666667 - }, - "0.01": { - "tp": 2328, - "fn": 72, - "accuracy": 0.97 - } - }, - "auroc": 0.971588888888889 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2348, - "fn": 52, - "accuracy": 0.9783333333333334 - }, - "0.01": { - "tp": 2267, - "fn": 133, - "accuracy": 0.9445833333333333 - } - }, - "auroc": 0.9597577256944445 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2348, - "fn": 52, - "accuracy": 0.9783333333333334 - }, - "0.01": { - "tp": 2267, - "fn": 133, - "accuracy": 0.9445833333333333 - } - }, - "auroc": 0.9597577256944445 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4731, - "fn": 69, - "accuracy": 0.985625 - }, - "0.01": { - "tp": 4595, - "fn": 205, - "accuracy": 0.9572916666666667 - } - }, - "auroc": 0.9656733072916668 - }, - { - "domain": "news", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4731, - "fn": 69, - "accuracy": 0.985625 - }, - "0.01": { - "tp": 4595, - "fn": 205, - "accuracy": 0.9572916666666667 - } - }, - "auroc": 0.9656733072916668 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2348, - "fn": 52, - "accuracy": 0.9783333333333334 - }, - "0.01": { - "tp": 2130, - "fn": 270, - "accuracy": 0.8875 - } - }, - "auroc": 0.9329148437500001 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2348, - "fn": 52, - "accuracy": 0.9783333333333334 - }, - "0.01": { - "tp": 2130, - "fn": 270, - "accuracy": 0.8875 - } - }, - "auroc": 0.9329148437500001 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2232, - "fn": 168, - "accuracy": 0.93 - }, - "0.01": { - "tp": 1878, - "fn": 522, - "accuracy": 0.7825 - } - }, - "auroc": 0.8987944444444446 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2232, - "fn": 168, - "accuracy": 0.93 - }, - "0.01": { - "tp": 1878, - "fn": 522, - "accuracy": 0.7825 - } - }, - "auroc": 0.8987944444444446 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4580, - "fn": 220, - "accuracy": 0.9541666666666667 - }, - "0.01": { - "tp": 4008, - "fn": 792, - "accuracy": 0.835 - } - }, - "auroc": 0.9158546440972222 - }, - { - "domain": "news", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4580, - "fn": 220, - "accuracy": 0.9541666666666667 - }, - "0.01": { - "tp": 4008, - "fn": 792, - "accuracy": 0.835 - } - }, - "auroc": 0.9158546440972222 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9912591145833334 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9912591145833334 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9904875868055556 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9904875868055556 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4792, - "fn": 8, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9908733506944444 - }, - { - "domain": "news", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4792, - "fn": 8, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9908733506944444 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9912788194444445 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9912788194444445 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 2387, - "fn": 13, - "accuracy": 0.9945833333333334 - } - }, - "auroc": 0.9801348958333334 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 2387, - "fn": 13, - "accuracy": 0.9945833333333334 - } - }, - "auroc": 0.9801348958333334 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4798, - "fn": 2, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 4786, - "fn": 14, - "accuracy": 0.9970833333333333 - } - }, - "auroc": 0.9857068576388889 - }, - { - "domain": "news", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4798, - "fn": 2, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 4786, - "fn": 14, - "accuracy": 0.9970833333333333 - } - }, - "auroc": 0.9857068576388889 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2319, - "fn": 81, - "accuracy": 0.96625 - }, - "0.01": { - "tp": 2162, - "fn": 238, - "accuracy": 0.9008333333333334 - } - }, - "auroc": 0.9490538194444444 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2319, - "fn": 81, - "accuracy": 0.96625 - }, - "0.01": { - "tp": 2162, - "fn": 238, - "accuracy": 0.9008333333333334 - } - }, - "auroc": 0.9490538194444444 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2325, - "fn": 75, - "accuracy": 0.96875 - }, - "0.01": { - "tp": 2144, - "fn": 256, - "accuracy": 0.8933333333333333 - } - }, - "auroc": 0.940617013888889 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2325, - "fn": 75, - "accuracy": 0.96875 - }, - "0.01": { - "tp": 2144, - "fn": 256, - "accuracy": 0.8933333333333333 - } - }, - "auroc": 0.940617013888889 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4644, - "fn": 156, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 4306, - "fn": 494, - "accuracy": 0.8970833333333333 - } - }, - "auroc": 0.9448354166666668 - }, - { - "domain": "news", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4644, - "fn": 156, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 4306, - "fn": 494, - "accuracy": 0.8970833333333333 - } - }, - "auroc": 0.9448354166666668 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 26138, - "fn": 262, - "accuracy": 0.9900757575757576 - }, - "0.01": { - "tp": 25562, - "fn": 838, - "accuracy": 0.9682575757575758 - } - }, - "auroc": 0.9752489425505051 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 14201, - "fn": 199, - "accuracy": 0.9861805555555555 - }, - "0.01": { - "tp": 13942, - "fn": 458, - "accuracy": 0.9681944444444445 - } - }, - "auroc": 0.9761049334490741 - }, - { - "domain": "news", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 40339, - "fn": 461, - "accuracy": 0.9887009803921568 - }, - "0.01": { - "tp": 39504, - "fn": 1296, - "accuracy": 0.9682352941176471 - } - }, - "auroc": 0.9755510569852941 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 25158, - "fn": 1242, - "accuracy": 0.9529545454545455 - }, - "0.01": { - "tp": 23251, - "fn": 3149, - "accuracy": 0.880719696969697 - } - }, - "auroc": 0.9397369239267677 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 13952, - "fn": 448, - "accuracy": 0.9688888888888889 - }, - "0.01": { - "tp": 13487, - "fn": 913, - "accuracy": 0.9365972222222222 - } - }, - "auroc": 0.9554916666666667 - }, - { - "domain": "news", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 39110, - "fn": 1690, - "accuracy": 0.9585784313725491 - }, - "0.01": { - "tp": 36738, - "fn": 4062, - "accuracy": 0.9004411764705882 - } - }, - "auroc": 0.945297421364379 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 51296, - "fn": 1504, - "accuracy": 0.9715151515151516 - }, - "0.01": { - "tp": 48813, - "fn": 3987, - "accuracy": 0.9244886363636363 - } - }, - "auroc": 0.9574929332386364 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 28153, - "fn": 647, - "accuracy": 0.9775347222222223 - }, - "0.01": { - "tp": 27429, - "fn": 1371, - "accuracy": 0.9523958333333333 - } - }, - "auroc": 0.9657983000578703 - }, - { - "domain": "news", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 79449, - "fn": 2151, - "accuracy": 0.9736397058823529 - }, - "0.01": { - "tp": 76242, - "fn": 5358, - "accuracy": 0.9343382352941176 - } - }, - "auroc": 0.9604242391748367 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9866395833333332 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9805291666666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9835843750000001 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9838333333333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.974709375 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9792713541666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9852364583333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9776192708333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 789, - "fn": 11, - "accuracy": 0.98625 - } - }, - "auroc": 0.9814278645833334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - }, - "0.01": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - } - }, - "auroc": 0.9356895833333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9832208333333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 302, - "fn": 98, - "accuracy": 0.755 - } - }, - "auroc": 0.9594552083333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9706833333333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9672020833333332 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 344, - "fn": 56, - "accuracy": 0.86 - } - }, - "auroc": 0.9689427083333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 291, - "fn": 109, - "accuracy": 0.7275 - } - }, - "auroc": 0.9531864583333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - } - }, - "auroc": 0.9752114583333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 741, - "fn": 59, - "accuracy": 0.92625 - }, - "0.01": { - "tp": 646, - "fn": 154, - "accuracy": 0.8075 - } - }, - "auroc": 0.9641989583333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9841520833333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9761249999999999 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9801385416666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9822458333333333 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 160, - "fn": 40, - "accuracy": 0.8 - } - }, - "auroc": 0.9591822916666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9707140624999999 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9831989583333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9676536458333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 748, - "fn": 52, - "accuracy": 0.935 - } - }, - "auroc": 0.9754263020833334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 132, - "fn": 68, - "accuracy": 0.66 - }, - "0.01": { - "tp": 85, - "fn": 115, - "accuracy": 0.425 - } - }, - "auroc": 0.8987333333333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.98414375 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 327, - "fn": 73, - "accuracy": 0.8175 - }, - "0.01": { - "tp": 266, - "fn": 134, - "accuracy": 0.665 - } - }, - "auroc": 0.9414385416666666 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.9512427083333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9665447916666667 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 324, - "fn": 76, - "accuracy": 0.81 - } - }, - "auroc": 0.9588937500000001 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 314, - "fn": 86, - "accuracy": 0.785 - }, - "0.01": { - "tp": 232, - "fn": 168, - "accuracy": 0.58 - } - }, - "auroc": 0.9249880208333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9753442708333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 700, - "fn": 100, - "accuracy": 0.875 - }, - "0.01": { - "tp": 590, - "fn": 210, - "accuracy": 0.7375 - } - }, - "auroc": 0.9501661458333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - }, - "0.01": { - "tp": 111, - "fn": 89, - "accuracy": 0.555 - } - }, - "auroc": 0.9250666666666666 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9865312500000001 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 306, - "fn": 94, - "accuracy": 0.765 - } - }, - "auroc": 0.9557989583333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - }, - "0.01": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - } - }, - "auroc": 0.913596875 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 159, - "fn": 41, - "accuracy": 0.795 - } - }, - "auroc": 0.9619197916666666 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - }, - "0.01": { - "tp": 278, - "fn": 122, - "accuracy": 0.695 - } - }, - "auroc": 0.9377583333333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - }, - "0.01": { - "tp": 230, - "fn": 170, - "accuracy": 0.575 - } - }, - "auroc": 0.9193317708333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9742255208333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 715, - "fn": 85, - "accuracy": 0.89375 - }, - "0.01": { - "tp": 584, - "fn": 216, - "accuracy": 0.73 - } - }, - "auroc": 0.9467786458333334 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9877218750000001 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9817979166666666 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9847598958333332 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9834010416666668 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.970953125 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9771770833333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9855614583333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9763755208333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 780, - "fn": 20, - "accuracy": 0.975 - } - }, - "auroc": 0.9809684895833334 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9586385416666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9586385416666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9531302083333333 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9531302083333333 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - } - }, - "auroc": 0.9558843750000001 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - } - }, - "auroc": 0.9558843750000001 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9538979166666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9538979166666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - } - }, - "auroc": 0.9304760416666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - } - }, - "auroc": 0.9304760416666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - }, - "0.01": { - "tp": 299, - "fn": 101, - "accuracy": 0.7475 - } - }, - "auroc": 0.9421869791666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - }, - "0.01": { - "tp": 299, - "fn": 101, - "accuracy": 0.7475 - } - }, - "auroc": 0.9421869791666667 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9659385416666668 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9659385416666668 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9590291666666666 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9590291666666666 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9624838541666667 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9624838541666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9771541666666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9771541666666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9526895833333333 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9526895833333333 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9649218749999999 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9649218749999999 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9791541666666667 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9791541666666667 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9551802083333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9551802083333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9671671875000001 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9671671875000001 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2037, - "fn": 163, - "accuracy": 0.9259090909090909 - }, - "0.01": { - "tp": 1833, - "fn": 367, - "accuracy": 0.8331818181818181 - } - }, - "auroc": 0.9593442234848484 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1181, - "fn": 19, - "accuracy": 0.9841666666666666 - }, - "0.01": { - "tp": 1148, - "fn": 52, - "accuracy": 0.9566666666666667 - } - }, - "auroc": 0.9820579861111111 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3218, - "fn": 182, - "accuracy": 0.9464705882352941 - }, - "0.01": { - "tp": 2981, - "fn": 419, - "accuracy": 0.8767647058823529 - } - }, - "auroc": 0.9673608455882354 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2086, - "fn": 114, - "accuracy": 0.9481818181818182 - }, - "0.01": { - "tp": 1866, - "fn": 334, - "accuracy": 0.8481818181818181 - } - }, - "auroc": 0.9577734848484849 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1154, - "fn": 46, - "accuracy": 0.9616666666666667 - }, - "0.01": { - "tp": 1045, - "fn": 155, - "accuracy": 0.8708333333333333 - } - }, - "auroc": 0.9667519097222222 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3240, - "fn": 160, - "accuracy": 0.9529411764705882 - }, - "0.01": { - "tp": 2911, - "fn": 489, - "accuracy": 0.8561764705882353 - } - }, - "auroc": 0.9609423406862745 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 4123, - "fn": 277, - "accuracy": 0.9370454545454545 - }, - "0.01": { - "tp": 3699, - "fn": 701, - "accuracy": 0.8406818181818182 - } - }, - "auroc": 0.9585588541666666 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2335, - "fn": 65, - "accuracy": 0.9729166666666667 - }, - "0.01": { - "tp": 2193, - "fn": 207, - "accuracy": 0.91375 - } - }, - "auroc": 0.9744049479166668 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 6458, - "fn": 342, - "accuracy": 0.9497058823529412 - }, - "0.01": { - "tp": 5892, - "fn": 908, - "accuracy": 0.8664705882352941 - } - }, - "auroc": 0.964151593137255 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9866395833333332 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9805291666666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9835843750000001 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9838333333333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.974709375 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9792713541666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9852364583333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9776192708333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 789, - "fn": 11, - "accuracy": 0.98625 - } - }, - "auroc": 0.9814278645833334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - }, - "0.01": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - } - }, - "auroc": 0.9356895833333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9832208333333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 302, - "fn": 98, - "accuracy": 0.755 - } - }, - "auroc": 0.9594552083333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9706833333333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9672020833333332 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 344, - "fn": 56, - "accuracy": 0.86 - } - }, - "auroc": 0.9689427083333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 291, - "fn": 109, - "accuracy": 0.7275 - } - }, - "auroc": 0.9531864583333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - } - }, - "auroc": 0.9752114583333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 741, - "fn": 59, - "accuracy": 0.92625 - }, - "0.01": { - "tp": 646, - "fn": 154, - "accuracy": 0.8075 - } - }, - "auroc": 0.9641989583333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9841520833333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9761249999999999 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9801385416666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9822458333333333 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 160, - "fn": 40, - "accuracy": 0.8 - } - }, - "auroc": 0.9591822916666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9707140624999999 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9831989583333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9676536458333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 748, - "fn": 52, - "accuracy": 0.935 - } - }, - "auroc": 0.9754263020833334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 132, - "fn": 68, - "accuracy": 0.66 - }, - "0.01": { - "tp": 85, - "fn": 115, - "accuracy": 0.425 - } - }, - "auroc": 0.8987333333333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.98414375 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 327, - "fn": 73, - "accuracy": 0.8175 - }, - "0.01": { - "tp": 266, - "fn": 134, - "accuracy": 0.665 - } - }, - "auroc": 0.9414385416666666 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.9512427083333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9665447916666667 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 324, - "fn": 76, - "accuracy": 0.81 - } - }, - "auroc": 0.9588937500000001 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 314, - "fn": 86, - "accuracy": 0.785 - }, - "0.01": { - "tp": 232, - "fn": 168, - "accuracy": 0.58 - } - }, - "auroc": 0.9249880208333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9753442708333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 700, - "fn": 100, - "accuracy": 0.875 - }, - "0.01": { - "tp": 590, - "fn": 210, - "accuracy": 0.7375 - } - }, - "auroc": 0.9501661458333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - }, - "0.01": { - "tp": 111, - "fn": 89, - "accuracy": 0.555 - } - }, - "auroc": 0.9250666666666666 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9865312500000001 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 306, - "fn": 94, - "accuracy": 0.765 - } - }, - "auroc": 0.9557989583333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - }, - "0.01": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - } - }, - "auroc": 0.913596875 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 159, - "fn": 41, - "accuracy": 0.795 - } - }, - "auroc": 0.9619197916666666 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - }, - "0.01": { - "tp": 278, - "fn": 122, - "accuracy": 0.695 - } - }, - "auroc": 0.9377583333333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - }, - "0.01": { - "tp": 230, - "fn": 170, - "accuracy": 0.575 - } - }, - "auroc": 0.9193317708333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9742255208333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 715, - "fn": 85, - "accuracy": 0.89375 - }, - "0.01": { - "tp": 584, - "fn": 216, - "accuracy": 0.73 - } - }, - "auroc": 0.9467786458333334 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9877218750000001 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9817979166666666 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9847598958333332 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9834010416666668 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.970953125 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9771770833333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9855614583333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9763755208333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 780, - "fn": 20, - "accuracy": 0.975 - } - }, - "auroc": 0.9809684895833334 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9586385416666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9586385416666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9531302083333333 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9531302083333333 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - } - }, - "auroc": 0.9558843750000001 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - } - }, - "auroc": 0.9558843750000001 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9538979166666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9538979166666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - } - }, - "auroc": 0.9304760416666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - } - }, - "auroc": 0.9304760416666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - }, - "0.01": { - "tp": 299, - "fn": 101, - "accuracy": 0.7475 - } - }, - "auroc": 0.9421869791666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - }, - "0.01": { - "tp": 299, - "fn": 101, - "accuracy": 0.7475 - } - }, - "auroc": 0.9421869791666667 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9659385416666668 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9659385416666668 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9590291666666666 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9590291666666666 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9624838541666667 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9624838541666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9771541666666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9771541666666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9526895833333333 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9526895833333333 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9649218749999999 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9649218749999999 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9791541666666667 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9791541666666667 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9551802083333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9551802083333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9671671875000001 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9671671875000001 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2037, - "fn": 163, - "accuracy": 0.9259090909090909 - }, - "0.01": { - "tp": 1833, - "fn": 367, - "accuracy": 0.8331818181818181 - } - }, - "auroc": 0.9593442234848484 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1181, - "fn": 19, - "accuracy": 0.9841666666666666 - }, - "0.01": { - "tp": 1148, - "fn": 52, - "accuracy": 0.9566666666666667 - } - }, - "auroc": 0.9820579861111111 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3218, - "fn": 182, - "accuracy": 0.9464705882352941 - }, - "0.01": { - "tp": 2981, - "fn": 419, - "accuracy": 0.8767647058823529 - } - }, - "auroc": 0.9673608455882354 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2086, - "fn": 114, - "accuracy": 0.9481818181818182 - }, - "0.01": { - "tp": 1866, - "fn": 334, - "accuracy": 0.8481818181818181 - } - }, - "auroc": 0.9577734848484849 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1154, - "fn": 46, - "accuracy": 0.9616666666666667 - }, - "0.01": { - "tp": 1045, - "fn": 155, - "accuracy": 0.8708333333333333 - } - }, - "auroc": 0.9667519097222222 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3240, - "fn": 160, - "accuracy": 0.9529411764705882 - }, - "0.01": { - "tp": 2911, - "fn": 489, - "accuracy": 0.8561764705882353 - } - }, - "auroc": 0.9609423406862745 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 4123, - "fn": 277, - "accuracy": 0.9370454545454545 - }, - "0.01": { - "tp": 3699, - "fn": 701, - "accuracy": 0.8406818181818182 - } - }, - "auroc": 0.9585588541666666 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2335, - "fn": 65, - "accuracy": 0.9729166666666667 - }, - "0.01": { - "tp": 2193, - "fn": 207, - "accuracy": 0.91375 - } - }, - "auroc": 0.9744049479166668 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 6458, - "fn": 342, - "accuracy": 0.9497058823529412 - }, - "0.01": { - "tp": 5892, - "fn": 908, - "accuracy": 0.8664705882352941 - } - }, - "auroc": 0.9641515931372548 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9819791666666666 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9769072916666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9794432291666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.980096875 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9714072916666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9757520833333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9810380208333334 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9741572916666668 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 788, - "fn": 12, - "accuracy": 0.985 - } - }, - "auroc": 0.9775976562500001 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9651583333333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.984334375 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 348, - "fn": 52, - "accuracy": 0.87 - } - }, - "auroc": 0.9747463541666668 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9807885416666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9670177083333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9739031249999999 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 349, - "fn": 51, - "accuracy": 0.8725 - } - }, - "auroc": 0.9729734375 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9756760416666665 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 765, - "fn": 35, - "accuracy": 0.95625 - }, - "0.01": { - "tp": 705, - "fn": 95, - "accuracy": 0.88125 - } - }, - "auroc": 0.9743247395833334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9807427083333333 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9728729166666665 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9768078124999999 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9783072916666666 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 160, - "fn": 40, - "accuracy": 0.8 - } - }, - "auroc": 0.9564687499999999 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9673880208333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.979525 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9646708333333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 745, - "fn": 55, - "accuracy": 0.93125 - } - }, - "auroc": 0.9720979166666668 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 156, - "fn": 44, - "accuracy": 0.78 - }, - "0.01": { - "tp": 121, - "fn": 79, - "accuracy": 0.605 - } - }, - "auroc": 0.9220031249999999 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9922145833333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - }, - "0.01": { - "tp": 315, - "fn": 85, - "accuracy": 0.7875 - } - }, - "auroc": 0.9571088541666666 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9682614583333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9774427083333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9728520833333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 348, - "fn": 52, - "accuracy": 0.87 - }, - "0.01": { - "tp": 293, - "fn": 107, - "accuracy": 0.7325 - } - }, - "auroc": 0.9451322916666667 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9848286458333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 740, - "fn": 60, - "accuracy": 0.925 - }, - "0.01": { - "tp": 672, - "fn": 128, - "accuracy": 0.84 - } - }, - "auroc": 0.9649804687499999 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 156, - "fn": 44, - "accuracy": 0.78 - } - }, - "auroc": 0.9574281250000001 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9886447916666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9730364583333333 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.9358364583333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - } - }, - "auroc": 0.96635625 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 314, - "fn": 86, - "accuracy": 0.785 - } - }, - "auroc": 0.9510963541666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - }, - "0.01": { - "tp": 303, - "fn": 97, - "accuracy": 0.7575 - } - }, - "auroc": 0.9466322916666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - } - }, - "auroc": 0.9775005208333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 755, - "fn": 45, - "accuracy": 0.94375 - }, - "0.01": { - "tp": 664, - "fn": 136, - "accuracy": 0.83 - } - }, - "auroc": 0.96206640625 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9835458333333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9764729166666666 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.980009375 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.97895625 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9663510416666667 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9726536458333334 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9812510416666667 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9714119791666667 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 778, - "fn": 22, - "accuracy": 0.9725 - } - }, - "auroc": 0.9763315104166667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9650020833333334 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9650020833333334 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9569791666666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9569791666666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 341, - "fn": 59, - "accuracy": 0.8525 - } - }, - "auroc": 0.9609906250000001 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 341, - "fn": 59, - "accuracy": 0.8525 - } - }, - "auroc": 0.9609906250000001 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9556604166666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9556604166666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 136, - "fn": 64, - "accuracy": 0.68 - } - }, - "auroc": 0.9366354166666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 136, - "fn": 64, - "accuracy": 0.68 - } - }, - "auroc": 0.9366354166666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 304, - "fn": 96, - "accuracy": 0.76 - } - }, - "auroc": 0.9461479166666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 304, - "fn": 96, - "accuracy": 0.76 - } - }, - "auroc": 0.9461479166666666 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9606739583333334 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9606739583333334 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9572291666666668 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9572291666666668 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9589515625 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9589515625 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9746645833333334 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9746645833333334 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9548635416666666 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9548635416666666 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9647640625 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9647640625 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9763031249999999 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9763031249999999 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.9553354166666668 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.9553354166666668 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 346, - "fn": 54, - "accuracy": 0.865 - } - }, - "auroc": 0.9658192708333333 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 346, - "fn": 54, - "accuracy": 0.865 - } - }, - "auroc": 0.9658192708333333 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2110, - "fn": 90, - "accuracy": 0.9590909090909091 - }, - "0.01": { - "tp": 1967, - "fn": 233, - "accuracy": 0.894090909090909 - } - }, - "auroc": 0.9657419507575757 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1185, - "fn": 15, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 1158, - "fn": 42, - "accuracy": 0.965 - } - }, - "auroc": 0.9819078124999999 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3295, - "fn": 105, - "accuracy": 0.9691176470588235 - }, - "0.01": { - "tp": 3125, - "fn": 275, - "accuracy": 0.9191176470588235 - } - }, - "auroc": 0.9714475490196078 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2124, - "fn": 76, - "accuracy": 0.9654545454545455 - }, - "0.01": { - "tp": 1936, - "fn": 264, - "accuracy": 0.88 - } - }, - "auroc": 0.9621172348484849 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1161, - "fn": 39, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 1061, - "fn": 139, - "accuracy": 0.8841666666666667 - } - }, - "auroc": 0.9675072916666667 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3285, - "fn": 115, - "accuracy": 0.9661764705882353 - }, - "0.01": { - "tp": 2997, - "fn": 403, - "accuracy": 0.8814705882352941 - } - }, - "auroc": 0.9640196078431372 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 4234, - "fn": 166, - "accuracy": 0.9622727272727273 - }, - "0.01": { - "tp": 3903, - "fn": 497, - "accuracy": 0.8870454545454546 - } - }, - "auroc": 0.9639295928030305 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2346, - "fn": 54, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 2219, - "fn": 181, - "accuracy": 0.9245833333333333 - } - }, - "auroc": 0.9747075520833334 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 6580, - "fn": 220, - "accuracy": 0.9676470588235294 - }, - "0.01": { - "tp": 6122, - "fn": 678, - "accuracy": 0.9002941176470588 - } - }, - "auroc": 0.9677335784313725 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9820656249999999 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9749947916666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9785302083333334 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.979696875 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9691614583333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9744291666666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9808812499999999 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9720781250000001 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 778, - "fn": 22, - "accuracy": 0.9725 - } - }, - "auroc": 0.9764796874999999 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - }, - "0.01": { - "tp": 118, - "fn": 82, - "accuracy": 0.59 - } - }, - "auroc": 0.9342166666666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9774510416666666 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 359, - "fn": 41, - "accuracy": 0.8975 - }, - "0.01": { - "tp": 295, - "fn": 105, - "accuracy": 0.7375 - } - }, - "auroc": 0.9558338541666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - } - }, - "auroc": 0.9614781250000001 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9623322916666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - }, - "0.01": { - "tp": 335, - "fn": 65, - "accuracy": 0.8375 - } - }, - "auroc": 0.9619052083333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - }, - "0.01": { - "tp": 284, - "fn": 116, - "accuracy": 0.71 - } - }, - "auroc": 0.9478473958333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 346, - "fn": 54, - "accuracy": 0.865 - } - }, - "auroc": 0.9698916666666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 728, - "fn": 72, - "accuracy": 0.91 - }, - "0.01": { - "tp": 630, - "fn": 170, - "accuracy": 0.7875 - } - }, - "auroc": 0.95886953125 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9803375000000001 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.964746875 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9725421875 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9770635416666668 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 145, - "fn": 55, - "accuracy": 0.725 - } - }, - "auroc": 0.9487979166666666 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 338, - "fn": 62, - "accuracy": 0.845 - } - }, - "auroc": 0.9629307291666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9787005208333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 329, - "fn": 71, - "accuracy": 0.8225 - } - }, - "auroc": 0.9567723958333333 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 786, - "fn": 14, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 720, - "fn": 80, - "accuracy": 0.9 - } - }, - "auroc": 0.9677364583333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 133, - "fn": 67, - "accuracy": 0.665 - }, - "0.01": { - "tp": 83, - "fn": 117, - "accuracy": 0.415 - } - }, - "auroc": 0.8981208333333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9780239583333332 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - }, - "0.01": { - "tp": 259, - "fn": 141, - "accuracy": 0.6475 - } - }, - "auroc": 0.9380723958333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - }, - "0.01": { - "tp": 122, - "fn": 78, - "accuracy": 0.61 - } - }, - "auroc": 0.9311677083333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.9535562499999999 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 280, - "fn": 120, - "accuracy": 0.7 - } - }, - "auroc": 0.9423619791666666 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 309, - "fn": 91, - "accuracy": 0.7725 - }, - "0.01": { - "tp": 205, - "fn": 195, - "accuracy": 0.5125 - } - }, - "auroc": 0.9146442708333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 334, - "fn": 66, - "accuracy": 0.835 - } - }, - "auroc": 0.9657901041666667 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 691, - "fn": 109, - "accuracy": 0.86375 - }, - "0.01": { - "tp": 539, - "fn": 261, - "accuracy": 0.67375 - } - }, - "auroc": 0.9402171875 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - }, - "0.01": { - "tp": 107, - "fn": 93, - "accuracy": 0.535 - } - }, - "auroc": 0.9206677083333333 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9726864583333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - }, - "0.01": { - "tp": 286, - "fn": 114, - "accuracy": 0.715 - } - }, - "auroc": 0.9466770833333333 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - }, - "0.01": { - "tp": 104, - "fn": 96, - "accuracy": 0.52 - } - }, - "auroc": 0.8994395833333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 155, - "fn": 45, - "accuracy": 0.775 - } - }, - "auroc": 0.9534031249999999 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 347, - "fn": 53, - "accuracy": 0.8675 - }, - "0.01": { - "tp": 259, - "fn": 141, - "accuracy": 0.6475 - } - }, - "auroc": 0.9264213541666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 327, - "fn": 73, - "accuracy": 0.8175 - }, - "0.01": { - "tp": 211, - "fn": 189, - "accuracy": 0.5275 - } - }, - "auroc": 0.9100536458333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 334, - "fn": 66, - "accuracy": 0.835 - } - }, - "auroc": 0.9630447916666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 703, - "fn": 97, - "accuracy": 0.87875 - }, - "0.01": { - "tp": 545, - "fn": 255, - "accuracy": 0.68125 - } - }, - "auroc": 0.9365492187500001 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.984125 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9750489583333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9795869791666667 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9758635416666667 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9566364583333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.96625 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9799942708333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9658427083333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 787, - "fn": 13, - "accuracy": 0.98375 - }, - "0.01": { - "tp": 755, - "fn": 45, - "accuracy": 0.94375 - } - }, - "auroc": 0.9729184895833334 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 152, - "fn": 48, - "accuracy": 0.76 - } - }, - "auroc": 0.9487479166666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 152, - "fn": 48, - "accuracy": 0.76 - } - }, - "auroc": 0.9487479166666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 145, - "fn": 55, - "accuracy": 0.725 - } - }, - "auroc": 0.9370458333333334 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 145, - "fn": 55, - "accuracy": 0.725 - } - }, - "auroc": 0.9370458333333334 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - }, - "0.01": { - "tp": 297, - "fn": 103, - "accuracy": 0.7425 - } - }, - "auroc": 0.9428968750000001 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - }, - "0.01": { - "tp": 297, - "fn": 103, - "accuracy": 0.7425 - } - }, - "auroc": 0.9428968750000001 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.94548125 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.94548125 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 125, - "fn": 75, - "accuracy": 0.625 - } - }, - "auroc": 0.9200875000000001 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 125, - "fn": 75, - "accuracy": 0.625 - } - }, - "auroc": 0.9200875000000001 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - }, - "0.01": { - "tp": 286, - "fn": 114, - "accuracy": 0.715 - } - }, - "auroc": 0.932784375 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - }, - "0.01": { - "tp": 286, - "fn": 114, - "accuracy": 0.715 - } - }, - "auroc": 0.932784375 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9602572916666667 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9602572916666667 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9546916666666666 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9546916666666666 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9574744791666667 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9574744791666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9755114583333333 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9755114583333333 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9537135416666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9537135416666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9646125000000001 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9646125000000001 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.968046875 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.968046875 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 145, - "fn": 55, - "accuracy": 0.725 - } - }, - "auroc": 0.9419843750000001 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 145, - "fn": 55, - "accuracy": 0.725 - } - }, - "auroc": 0.9419843750000001 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 319, - "fn": 81, - "accuracy": 0.7975 - } - }, - "auroc": 0.9550156250000001 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 319, - "fn": 81, - "accuracy": 0.7975 - } - }, - "auroc": 0.9550156250000001 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2023, - "fn": 177, - "accuracy": 0.9195454545454546 - }, - "0.01": { - "tp": 1783, - "fn": 417, - "accuracy": 0.8104545454545454 - } - }, - "auroc": 0.9543252840909091 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1172, - "fn": 28, - "accuracy": 0.9766666666666667 - }, - "0.01": { - "tp": 1106, - "fn": 94, - "accuracy": 0.9216666666666666 - } - }, - "auroc": 0.9738253472222222 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3195, - "fn": 205, - "accuracy": 0.9397058823529412 - }, - "0.01": { - "tp": 2889, - "fn": 511, - "accuracy": 0.8497058823529412 - } - }, - "auroc": 0.9612076593137253 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2052, - "fn": 148, - "accuracy": 0.9327272727272727 - }, - "0.01": { - "tp": 1765, - "fn": 435, - "accuracy": 0.8022727272727272 - } - }, - "auroc": 0.9483847537878788 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1136, - "fn": 64, - "accuracy": 0.9466666666666667 - }, - "0.01": { - "tp": 986, - "fn": 214, - "accuracy": 0.8216666666666667 - } - }, - "auroc": 0.9573145833333333 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3188, - "fn": 212, - "accuracy": 0.9376470588235294 - }, - "0.01": { - "tp": 2751, - "fn": 649, - "accuracy": 0.8091176470588235 - } - }, - "auroc": 0.9515364583333333 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 4075, - "fn": 325, - "accuracy": 0.9261363636363636 - }, - "0.01": { - "tp": 3548, - "fn": 852, - "accuracy": 0.8063636363636364 - } - }, - "auroc": 0.951355018939394 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2308, - "fn": 92, - "accuracy": 0.9616666666666667 - }, - "0.01": { - "tp": 2092, - "fn": 308, - "accuracy": 0.8716666666666667 - } - }, - "auroc": 0.9655699652777777 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 6383, - "fn": 417, - "accuracy": 0.9386764705882353 - }, - "0.01": { - "tp": 5640, - "fn": 1160, - "accuracy": 0.8294117647058824 - } - }, - "auroc": 0.9563720588235294 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9863447916666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9801593749999999 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9832520833333334 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.98348125 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9746999999999999 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.979090625 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9849130208333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9774296875 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - } - }, - "auroc": 0.9811713541666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - }, - "0.01": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - } - }, - "auroc": 0.9359770833333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.982075 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 301, - "fn": 99, - "accuracy": 0.7525 - } - }, - "auroc": 0.9590260416666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9775677083333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.9669395833333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 348, - "fn": 52, - "accuracy": 0.87 - } - }, - "auroc": 0.9722536458333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - }, - "0.01": { - "tp": 296, - "fn": 104, - "accuracy": 0.74 - } - }, - "auroc": 0.9567723958333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.9745072916666666 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 745, - "fn": 55, - "accuracy": 0.93125 - }, - "0.01": { - "tp": 649, - "fn": 151, - "accuracy": 0.81125 - } - }, - "auroc": 0.96563984375 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9841104166666668 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9758645833333333 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9799875 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9821916666666666 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9598416666666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9710166666666666 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9831510416666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.9678531250000001 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 748, - "fn": 52, - "accuracy": 0.935 - } - }, - "auroc": 0.9755020833333332 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 132, - "fn": 68, - "accuracy": 0.66 - }, - "0.01": { - "tp": 84, - "fn": 116, - "accuracy": 0.42 - } - }, - "auroc": 0.8986239583333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.984946875 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - }, - "0.01": { - "tp": 265, - "fn": 135, - "accuracy": 0.6625 - } - }, - "auroc": 0.9417854166666667 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 159, - "fn": 41, - "accuracy": 0.795 - } - }, - "auroc": 0.9620937500000001 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9724239583333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 340, - "fn": 60, - "accuracy": 0.85 - } - }, - "auroc": 0.9672588541666668 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 319, - "fn": 81, - "accuracy": 0.7975 - }, - "0.01": { - "tp": 243, - "fn": 157, - "accuracy": 0.6075 - } - }, - "auroc": 0.9303588541666666 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - } - }, - "auroc": 0.9786854166666666 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 707, - "fn": 93, - "accuracy": 0.88375 - }, - "0.01": { - "tp": 605, - "fn": 195, - "accuracy": 0.75625 - } - }, - "auroc": 0.9545221354166666 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - }, - "0.01": { - "tp": 112, - "fn": 88, - "accuracy": 0.56 - } - }, - "auroc": 0.9253031249999999 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9870062500000001 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 306, - "fn": 94, - "accuracy": 0.765 - } - }, - "auroc": 0.9561546875 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - }, - "0.01": { - "tp": 133, - "fn": 67, - "accuracy": 0.665 - } - }, - "auroc": 0.925709375 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9647197916666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 359, - "fn": 41, - "accuracy": 0.8975 - }, - "0.01": { - "tp": 297, - "fn": 103, - "accuracy": 0.7425 - } - }, - "auroc": 0.9452145833333333 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 339, - "fn": 61, - "accuracy": 0.8475 - }, - "0.01": { - "tp": 245, - "fn": 155, - "accuracy": 0.6125 - } - }, - "auroc": 0.92550625 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9758630208333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 722, - "fn": 78, - "accuracy": 0.9025 - }, - "0.01": { - "tp": 603, - "fn": 197, - "accuracy": 0.75375 - } - }, - "auroc": 0.9506846354166667 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9869406249999999 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9812749999999999 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9841078124999999 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9831593750000001 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9703875 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9767734375 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.98505 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9758312499999999 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 778, - "fn": 22, - "accuracy": 0.9725 - } - }, - "auroc": 0.980440625 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - } - }, - "auroc": 0.957328125 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - } - }, - "auroc": 0.957328125 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9522260416666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9522260416666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 325, - "fn": 75, - "accuracy": 0.8125 - } - }, - "auroc": 0.9547770833333333 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 325, - "fn": 75, - "accuracy": 0.8125 - } - }, - "auroc": 0.9547770833333333 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.95568125 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.95568125 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 135, - "fn": 65, - "accuracy": 0.675 - } - }, - "auroc": 0.933940625 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 135, - "fn": 65, - "accuracy": 0.675 - } - }, - "auroc": 0.933940625 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - }, - "0.01": { - "tp": 306, - "fn": 94, - "accuracy": 0.765 - } - }, - "auroc": 0.9448109374999999 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - }, - "0.01": { - "tp": 306, - "fn": 94, - "accuracy": 0.765 - } - }, - "auroc": 0.9448109374999999 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9654447916666665 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9654447916666665 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9584552083333333 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9584552083333333 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9619500000000001 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9619500000000001 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9767854166666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9767854166666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9539145833333333 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9539145833333333 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9653499999999999 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9653499999999999 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9792864583333333 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9792864583333333 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - } - }, - "auroc": 0.9545395833333332 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - } - }, - "auroc": 0.9545395833333332 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9669130208333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9669130208333334 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2038, - "fn": 162, - "accuracy": 0.9263636363636364 - }, - "0.01": { - "tp": 1834, - "fn": 366, - "accuracy": 0.8336363636363636 - } - }, - "auroc": 0.9592569128787878 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1182, - "fn": 18, - "accuracy": 0.985 - }, - "0.01": { - "tp": 1144, - "fn": 56, - "accuracy": 0.9533333333333334 - } - }, - "auroc": 0.9818878472222222 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3220, - "fn": 180, - "accuracy": 0.9470588235294117 - }, - "0.01": { - "tp": 2978, - "fn": 422, - "accuracy": 0.8758823529411764 - } - }, - "auroc": 0.9672443014705883 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2104, - "fn": 96, - "accuracy": 0.9563636363636364 - }, - "0.01": { - "tp": 1901, - "fn": 299, - "accuracy": 0.8640909090909091 - } - }, - "auroc": 0.9606617424242424 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1157, - "fn": 43, - "accuracy": 0.9641666666666666 - }, - "0.01": { - "tp": 1055, - "fn": 145, - "accuracy": 0.8791666666666667 - } - }, - "auroc": 0.9681687499999999 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3261, - "fn": 139, - "accuracy": 0.9591176470588235 - }, - "0.01": { - "tp": 2956, - "fn": 444, - "accuracy": 0.8694117647058823 - } - }, - "auroc": 0.963311274509804 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 4142, - "fn": 258, - "accuracy": 0.9413636363636364 - }, - "0.01": { - "tp": 3735, - "fn": 665, - "accuracy": 0.8488636363636364 - } - }, - "auroc": 0.9599593276515151 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2339, - "fn": 61, - "accuracy": 0.9745833333333334 - }, - "0.01": { - "tp": 2199, - "fn": 201, - "accuracy": 0.91625 - } - }, - "auroc": 0.9750282986111112 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 6481, - "fn": 319, - "accuracy": 0.9530882352941177 - }, - "0.01": { - "tp": 5934, - "fn": 866, - "accuracy": 0.8726470588235294 - } - }, - "auroc": 0.9652777879901961 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.994475 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.991384375 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9929296875 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.994659375 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.986425 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9905421875 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9945671875000001 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9889046875 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 783, - "fn": 17, - "accuracy": 0.97875 - } - }, - "auroc": 0.9917359375 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 152, - "fn": 48, - "accuracy": 0.76 - }, - "0.01": { - "tp": 78, - "fn": 122, - "accuracy": 0.39 - } - }, - "auroc": 0.8956927083333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 128, - "fn": 72, - "accuracy": 0.64 - } - }, - "auroc": 0.9330906250000001 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 327, - "fn": 73, - "accuracy": 0.8175 - }, - "0.01": { - "tp": 206, - "fn": 194, - "accuracy": 0.515 - } - }, - "auroc": 0.9143916666666666 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9661322916666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 145, - "fn": 55, - "accuracy": 0.725 - }, - "0.01": { - "tp": 102, - "fn": 98, - "accuracy": 0.51 - } - }, - "auroc": 0.8995531250000001 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 337, - "fn": 63, - "accuracy": 0.8425 - }, - "0.01": { - "tp": 275, - "fn": 125, - "accuracy": 0.6875 - } - }, - "auroc": 0.9328427083333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 344, - "fn": 56, - "accuracy": 0.86 - }, - "0.01": { - "tp": 251, - "fn": 149, - "accuracy": 0.6275 - } - }, - "auroc": 0.9309125 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 320, - "fn": 80, - "accuracy": 0.8 - }, - "0.01": { - "tp": 230, - "fn": 170, - "accuracy": 0.575 - } - }, - "auroc": 0.916321875 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 664, - "fn": 136, - "accuracy": 0.83 - }, - "0.01": { - "tp": 481, - "fn": 319, - "accuracy": 0.60125 - } - }, - "auroc": 0.9236171874999999 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9929583333333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.97521875 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9840885416666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9896125 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 146, - "fn": 54, - "accuracy": 0.73 - } - }, - "auroc": 0.9473572916666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 337, - "fn": 63, - "accuracy": 0.8425 - } - }, - "auroc": 0.9684848958333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9912854166666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 325, - "fn": 75, - "accuracy": 0.8125 - } - }, - "auroc": 0.9612880208333333 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 773, - "fn": 27, - "accuracy": 0.96625 - }, - "0.01": { - "tp": 712, - "fn": 88, - "accuracy": 0.89 - } - }, - "auroc": 0.9762867187500001 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 115, - "fn": 85, - "accuracy": 0.575 - }, - "0.01": { - "tp": 50, - "fn": 150, - "accuracy": 0.25 - } - }, - "auroc": 0.8640135416666667 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9763562499999999 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 304, - "fn": 96, - "accuracy": 0.76 - }, - "0.01": { - "tp": 229, - "fn": 171, - "accuracy": 0.5725 - } - }, - "auroc": 0.9201848958333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - } - }, - "auroc": 0.962746875 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 127, - "fn": 73, - "accuracy": 0.635 - } - }, - "auroc": 0.9364375 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 290, - "fn": 110, - "accuracy": 0.725 - } - }, - "auroc": 0.9495921875 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 306, - "fn": 94, - "accuracy": 0.765 - }, - "0.01": { - "tp": 213, - "fn": 187, - "accuracy": 0.5325 - } - }, - "auroc": 0.9133802083333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 306, - "fn": 94, - "accuracy": 0.765 - } - }, - "auroc": 0.956396875 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 681, - "fn": 119, - "accuracy": 0.85125 - }, - "0.01": { - "tp": 519, - "fn": 281, - "accuracy": 0.64875 - } - }, - "auroc": 0.9348885416666666 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 149, - "fn": 51, - "accuracy": 0.745 - }, - "0.01": { - "tp": 75, - "fn": 125, - "accuracy": 0.375 - } - }, - "auroc": 0.8930218749999999 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9691 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - }, - "0.01": { - "tp": 243, - "fn": 157, - "accuracy": 0.6075 - } - }, - "auroc": 0.9310609375000001 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.9568291666666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - }, - "0.01": { - "tp": 105, - "fn": 95, - "accuracy": 0.525 - } - }, - "auroc": 0.9193895833333333 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - }, - "0.01": { - "tp": 263, - "fn": 137, - "accuracy": 0.6575 - } - }, - "auroc": 0.9381093749999999 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 336, - "fn": 64, - "accuracy": 0.84 - }, - "0.01": { - "tp": 233, - "fn": 167, - "accuracy": 0.5825 - } - }, - "auroc": 0.9249255208333332 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - }, - "0.01": { - "tp": 273, - "fn": 127, - "accuracy": 0.6825 - } - }, - "auroc": 0.9442447916666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 702, - "fn": 98, - "accuracy": 0.8775 - }, - "0.01": { - "tp": 506, - "fn": 294, - "accuracy": 0.6325 - } - }, - "auroc": 0.93458515625 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9960885416666666 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9884208333333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9922546875 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9925520833333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9755385416666666 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9840453124999999 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9943203125 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9819796875 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - } - }, - "auroc": 0.9881500000000001 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - }, - "0.01": { - "tp": 140, - "fn": 60, - "accuracy": 0.7 - } - }, - "auroc": 0.9393541666666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - }, - "0.01": { - "tp": 140, - "fn": 60, - "accuracy": 0.7 - } - }, - "auroc": 0.9393541666666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 141, - "fn": 59, - "accuracy": 0.705 - } - }, - "auroc": 0.9421260416666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 141, - "fn": 59, - "accuracy": 0.705 - } - }, - "auroc": 0.9421260416666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - }, - "0.01": { - "tp": 281, - "fn": 119, - "accuracy": 0.7025 - } - }, - "auroc": 0.9407401041666666 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - }, - "0.01": { - "tp": 281, - "fn": 119, - "accuracy": 0.7025 - } - }, - "auroc": 0.9407401041666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9627114583333333 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9627114583333333 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 149, - "fn": 51, - "accuracy": 0.745 - } - }, - "auroc": 0.947234375 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 149, - "fn": 51, - "accuracy": 0.745 - } - }, - "auroc": 0.947234375 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - }, - "0.01": { - "tp": 313, - "fn": 87, - "accuracy": 0.7825 - } - }, - "auroc": 0.9549729166666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - }, - "0.01": { - "tp": 313, - "fn": 87, - "accuracy": 0.7825 - } - }, - "auroc": 0.9549729166666666 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9932937500000001 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9932937500000001 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9931614583333332 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9931614583333332 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9932276041666668 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9932276041666668 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9975822916666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9975822916666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9808989583333333 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9808989583333333 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.989240625 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.989240625 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9781479166666667 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9781479166666667 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - } - }, - "auroc": 0.9674260416666667 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - } - }, - "auroc": 0.9674260416666667 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 347, - "fn": 53, - "accuracy": 0.8675 - } - }, - "auroc": 0.9727869791666667 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 347, - "fn": 53, - "accuracy": 0.8675 - } - }, - "auroc": 0.9727869791666667 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1970, - "fn": 230, - "accuracy": 0.8954545454545455 - }, - "0.01": { - "tp": 1676, - "fn": 524, - "accuracy": 0.7618181818181818 - } - }, - "auroc": 0.9552126893939394 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1151, - "fn": 49, - "accuracy": 0.9591666666666666 - }, - "0.01": { - "tp": 1043, - "fn": 157, - "accuracy": 0.8691666666666666 - } - }, - "auroc": 0.9722618055555556 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3121, - "fn": 279, - "accuracy": 0.9179411764705883 - }, - "0.01": { - "tp": 2719, - "fn": 681, - "accuracy": 0.7997058823529412 - } - }, - "auroc": 0.9612300245098039 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2113, - "fn": 87, - "accuracy": 0.9604545454545454 - }, - "0.01": { - "tp": 1914, - "fn": 286, - "accuracy": 0.87 - } - }, - "auroc": 0.9721253787878787 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1076, - "fn": 124, - "accuracy": 0.8966666666666666 - }, - "0.01": { - "tp": 853, - "fn": 347, - "accuracy": 0.7108333333333333 - } - }, - "auroc": 0.9441168402777778 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3189, - "fn": 211, - "accuracy": 0.9379411764705883 - }, - "0.01": { - "tp": 2767, - "fn": 633, - "accuracy": 0.8138235294117647 - } - }, - "auroc": 0.9622400122549021 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 4083, - "fn": 317, - "accuracy": 0.9279545454545455 - }, - "0.01": { - "tp": 3590, - "fn": 810, - "accuracy": 0.8159090909090909 - } - }, - "auroc": 0.9636690340909091 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2227, - "fn": 173, - "accuracy": 0.9279166666666666 - }, - "0.01": { - "tp": 1896, - "fn": 504, - "accuracy": 0.79 - } - }, - "auroc": 0.9581893229166667 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 6310, - "fn": 490, - "accuracy": 0.9279411764705883 - }, - "0.01": { - "tp": 5486, - "fn": 1314, - "accuracy": 0.8067647058823529 - } - }, - "auroc": 0.961735018382353 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9866375 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9805333333333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9835854166666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9838333333333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.974709375 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9792713541666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9852354166666666 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9776213541666666 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 789, - "fn": 11, - "accuracy": 0.98625 - } - }, - "auroc": 0.9814283854166667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - }, - "0.01": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - } - }, - "auroc": 0.9357125 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9832145833333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 302, - "fn": 98, - "accuracy": 0.755 - } - }, - "auroc": 0.9594635416666666 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9724593749999999 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.9677364583333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 344, - "fn": 56, - "accuracy": 0.86 - } - }, - "auroc": 0.9700979166666668 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 292, - "fn": 108, - "accuracy": 0.73 - } - }, - "auroc": 0.9540859374999999 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9754755208333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 741, - "fn": 59, - "accuracy": 0.92625 - }, - "0.01": { - "tp": 646, - "fn": 154, - "accuracy": 0.8075 - } - }, - "auroc": 0.9647807291666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9841520833333333 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9761437499999999 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9801479166666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9822656249999999 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 160, - "fn": 40, - "accuracy": 0.8 - } - }, - "auroc": 0.9592395833333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9707526041666668 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9832088541666666 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9676916666666666 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 748, - "fn": 52, - "accuracy": 0.935 - } - }, - "auroc": 0.9754502604166667 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 132, - "fn": 68, - "accuracy": 0.66 - }, - "0.01": { - "tp": 85, - "fn": 115, - "accuracy": 0.425 - } - }, - "auroc": 0.8983260416666667 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.98415 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 327, - "fn": 73, - "accuracy": 0.8175 - }, - "0.01": { - "tp": 266, - "fn": 134, - "accuracy": 0.665 - } - }, - "auroc": 0.9412380208333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - }, - "0.01": { - "tp": 148, - "fn": 52, - "accuracy": 0.74 - } - }, - "auroc": 0.9521083333333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9675270833333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 325, - "fn": 75, - "accuracy": 0.8125 - } - }, - "auroc": 0.9598177083333335 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 314, - "fn": 86, - "accuracy": 0.785 - }, - "0.01": { - "tp": 233, - "fn": 167, - "accuracy": 0.5825 - } - }, - "auroc": 0.9252171875 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9758385416666666 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 700, - "fn": 100, - "accuracy": 0.875 - }, - "0.01": { - "tp": 591, - "fn": 209, - "accuracy": 0.73875 - } - }, - "auroc": 0.9505278645833334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - }, - "0.01": { - "tp": 111, - "fn": 89, - "accuracy": 0.555 - } - }, - "auroc": 0.9251510416666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9865802083333333 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 306, - "fn": 94, - "accuracy": 0.765 - } - }, - "auroc": 0.955865625 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - }, - "0.01": { - "tp": 122, - "fn": 78, - "accuracy": 0.61 - } - }, - "auroc": 0.9143708333333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 159, - "fn": 41, - "accuracy": 0.795 - } - }, - "auroc": 0.962428125 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - }, - "0.01": { - "tp": 281, - "fn": 119, - "accuracy": 0.7025 - } - }, - "auroc": 0.9383994791666668 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - }, - "0.01": { - "tp": 233, - "fn": 167, - "accuracy": 0.5825 - } - }, - "auroc": 0.9197609375 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9745041666666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 715, - "fn": 85, - "accuracy": 0.89375 - }, - "0.01": { - "tp": 587, - "fn": 213, - "accuracy": 0.73375 - } - }, - "auroc": 0.9471325520833334 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9877218750000001 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9818020833333334 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9847619791666666 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9834020833333335 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9709822916666666 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9771921874999999 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9855619791666665 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9763921875 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 780, - "fn": 20, - "accuracy": 0.975 - } - }, - "auroc": 0.9809770833333333 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9588104166666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9588104166666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9532135416666666 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9532135416666666 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - } - }, - "auroc": 0.9560119791666668 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - } - }, - "auroc": 0.9560119791666668 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9539645833333333 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9539645833333333 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - } - }, - "auroc": 0.93025625 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - } - }, - "auroc": 0.93025625 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - }, - "0.01": { - "tp": 299, - "fn": 101, - "accuracy": 0.7475 - } - }, - "auroc": 0.9421104166666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - }, - "0.01": { - "tp": 299, - "fn": 101, - "accuracy": 0.7475 - } - }, - "auroc": 0.9421104166666667 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9659385416666668 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9659385416666668 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9590291666666666 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9590291666666666 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9624838541666667 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9624838541666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9771322916666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9771322916666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9526916666666666 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9526916666666666 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9649119791666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9649119791666667 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9791635416666666 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9791635416666666 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.9551270833333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.9551270833333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - } - }, - "auroc": 0.9671453125 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - } - }, - "auroc": 0.9671453125 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2037, - "fn": 163, - "accuracy": 0.9259090909090909 - }, - "0.01": { - "tp": 1833, - "fn": 367, - "accuracy": 0.8331818181818181 - } - }, - "auroc": 0.9593373106060606 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1181, - "fn": 19, - "accuracy": 0.9841666666666666 - }, - "0.01": { - "tp": 1148, - "fn": 52, - "accuracy": 0.9566666666666667 - } - }, - "auroc": 0.9820706597222223 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3218, - "fn": 182, - "accuracy": 0.9464705882352941 - }, - "0.01": { - "tp": 2981, - "fn": 419, - "accuracy": 0.8767647058823529 - } - }, - "auroc": 0.9673608455882352 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2086, - "fn": 114, - "accuracy": 0.9481818181818182 - }, - "0.01": { - "tp": 1870, - "fn": 330, - "accuracy": 0.85 - } - }, - "auroc": 0.9580688446969697 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1154, - "fn": 46, - "accuracy": 0.9616666666666667 - }, - "0.01": { - "tp": 1044, - "fn": 156, - "accuracy": 0.87 - } - }, - "auroc": 0.9671038194444445 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3240, - "fn": 160, - "accuracy": 0.9529411764705882 - }, - "0.01": { - "tp": 2914, - "fn": 486, - "accuracy": 0.8570588235294118 - } - }, - "auroc": 0.9612576593137255 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 4123, - "fn": 277, - "accuracy": 0.9370454545454545 - }, - "0.01": { - "tp": 3703, - "fn": 697, - "accuracy": 0.8415909090909091 - } - }, - "auroc": 0.9587030776515151 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2335, - "fn": 65, - "accuracy": 0.9729166666666667 - }, - "0.01": { - "tp": 2192, - "fn": 208, - "accuracy": 0.9133333333333333 - } - }, - "auroc": 0.9745872395833334 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 6458, - "fn": 342, - "accuracy": 0.9497058823529412 - }, - "0.01": { - "tp": 5895, - "fn": 905, - "accuracy": 0.8669117647058824 - } - }, - "auroc": 0.9643092524509804 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9955552083333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9898822916666665 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9927187500000001 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9946489583333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9832458333333334 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9889473958333334 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9951020833333334 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9865640625000001 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 788, - "fn": 12, - "accuracy": 0.985 - } - }, - "auroc": 0.9908330729166667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - }, - "0.01": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - } - }, - "auroc": 0.9356625000000001 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.983315625 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 302, - "fn": 98, - "accuracy": 0.755 - } - }, - "auroc": 0.9594890625 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9697291666666665 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.9671166666666666 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 343, - "fn": 57, - "accuracy": 0.8575 - } - }, - "auroc": 0.9684229166666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 291, - "fn": 109, - "accuracy": 0.7275 - } - }, - "auroc": 0.9526958333333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9752161458333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 741, - "fn": 59, - "accuracy": 0.92625 - }, - "0.01": { - "tp": 645, - "fn": 155, - "accuracy": 0.80625 - } - }, - "auroc": 0.9639559895833334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9952302083333333 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9822124999999999 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9887213541666666 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.992621875 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - } - }, - "auroc": 0.9604927083333333 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.9765572916666666 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9939260416666666 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9713526041666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 753, - "fn": 47, - "accuracy": 0.94125 - } - }, - "auroc": 0.9826393229166667 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 132, - "fn": 68, - "accuracy": 0.66 - }, - "0.01": { - "tp": 85, - "fn": 115, - "accuracy": 0.425 - } - }, - "auroc": 0.8987239583333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9841927083333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 327, - "fn": 73, - "accuracy": 0.8175 - }, - "0.01": { - "tp": 266, - "fn": 134, - "accuracy": 0.665 - } - }, - "auroc": 0.9414583333333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - }, - "0.01": { - "tp": 146, - "fn": 54, - "accuracy": 0.73 - } - }, - "auroc": 0.951078125 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9662343750000001 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 323, - "fn": 77, - "accuracy": 0.8075 - } - }, - "auroc": 0.9586562500000001 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 314, - "fn": 86, - "accuracy": 0.785 - }, - "0.01": { - "tp": 231, - "fn": 169, - "accuracy": 0.5775 - } - }, - "auroc": 0.9249010416666668 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9752135416666667 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 700, - "fn": 100, - "accuracy": 0.875 - }, - "0.01": { - "tp": 589, - "fn": 211, - "accuracy": 0.73625 - } - }, - "auroc": 0.9500572916666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - }, - "0.01": { - "tp": 112, - "fn": 88, - "accuracy": 0.56 - } - }, - "auroc": 0.9250520833333333 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9868812499999999 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 306, - "fn": 94, - "accuracy": 0.765 - } - }, - "auroc": 0.9559666666666666 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - }, - "0.01": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - } - }, - "auroc": 0.9162833333333333 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 159, - "fn": 41, - "accuracy": 0.795 - } - }, - "auroc": 0.96105 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - }, - "0.01": { - "tp": 278, - "fn": 122, - "accuracy": 0.695 - } - }, - "auroc": 0.9386666666666668 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 335, - "fn": 65, - "accuracy": 0.8375 - }, - "0.01": { - "tp": 231, - "fn": 169, - "accuracy": 0.5775 - } - }, - "auroc": 0.9206677083333332 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.973965625 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 718, - "fn": 82, - "accuracy": 0.8975 - }, - "0.01": { - "tp": 584, - "fn": 216, - "accuracy": 0.73 - } - }, - "auroc": 0.9473166666666667 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99809375 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9954760416666667 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9967848958333334 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9944489583333334 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9800541666666667 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9872515625 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9962713541666666 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9877651041666666 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 783, - "fn": 17, - "accuracy": 0.97875 - } - }, - "auroc": 0.9920182291666666 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9586739583333334 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9586739583333334 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9549229166666666 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9549229166666666 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 329, - "fn": 71, - "accuracy": 0.8225 - } - }, - "auroc": 0.9567984375 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 329, - "fn": 71, - "accuracy": 0.8225 - } - }, - "auroc": 0.9567984375 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9614333333333334 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9614333333333334 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 132, - "fn": 68, - "accuracy": 0.66 - } - }, - "auroc": 0.9359791666666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 132, - "fn": 68, - "accuracy": 0.66 - } - }, - "auroc": 0.9359791666666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - }, - "0.01": { - "tp": 304, - "fn": 96, - "accuracy": 0.76 - } - }, - "auroc": 0.94870625 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - }, - "0.01": { - "tp": 304, - "fn": 96, - "accuracy": 0.76 - } - }, - "auroc": 0.94870625 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9895604166666667 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9895604166666667 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9829364583333333 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9829364583333333 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9862484375 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9862484375 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983270833333333 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983270833333333 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9754593749999999 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9754593749999999 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9868932291666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9868932291666667 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9847052083333333 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9847052083333333 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9607625 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9607625 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9727338541666666 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9727338541666666 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2037, - "fn": 163, - "accuracy": 0.9259090909090909 - }, - "0.01": { - "tp": 1837, - "fn": 363, - "accuracy": 0.835 - } - }, - "auroc": 0.967365246212121 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1182, - "fn": 18, - "accuracy": 0.985 - }, - "0.01": { - "tp": 1150, - "fn": 50, - "accuracy": 0.9583333333333334 - } - }, - "auroc": 0.9869934027777778 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3219, - "fn": 181, - "accuracy": 0.946764705882353 - }, - "0.01": { - "tp": 2987, - "fn": 413, - "accuracy": 0.8785294117647059 - } - }, - "auroc": 0.974292830882353 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2090, - "fn": 110, - "accuracy": 0.95 - }, - "0.01": { - "tp": 1877, - "fn": 323, - "accuracy": 0.8531818181818182 - } - }, - "auroc": 0.9662609848484849 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1154, - "fn": 46, - "accuracy": 0.9616666666666667 - }, - "0.01": { - "tp": 1048, - "fn": 152, - "accuracy": 0.8733333333333333 - } - }, - "auroc": 0.9696989583333334 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3244, - "fn": 156, - "accuracy": 0.9541176470588235 - }, - "0.01": { - "tp": 2925, - "fn": 475, - "accuracy": 0.8602941176470589 - } - }, - "auroc": 0.9674743872549019 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 4127, - "fn": 273, - "accuracy": 0.9379545454545455 - }, - "0.01": { - "tp": 3714, - "fn": 686, - "accuracy": 0.8440909090909091 - } - }, - "auroc": 0.9668131155303029 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2336, - "fn": 64, - "accuracy": 0.9733333333333334 - }, - "0.01": { - "tp": 2198, - "fn": 202, - "accuracy": 0.9158333333333334 - } - }, - "auroc": 0.9783461805555556 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 6463, - "fn": 337, - "accuracy": 0.9504411764705882 - }, - "0.01": { - "tp": 5912, - "fn": 888, - "accuracy": 0.8694117647058823 - } - }, - "auroc": 0.9708836090686275 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9769135416666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9740729166666666 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9754932291666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.97580625 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9731291666666666 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9744677083333334 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9763598958333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9736010416666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - } - }, - "auroc": 0.9749804687499999 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - }, - "0.01": { - "tp": 126, - "fn": 74, - "accuracy": 0.63 - } - }, - "auroc": 0.9378062500000001 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9863260416666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 314, - "fn": 86, - "accuracy": 0.785 - } - }, - "auroc": 0.9620661458333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9942885416666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9748197916666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9845541666666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - }, - "0.01": { - "tp": 325, - "fn": 75, - "accuracy": 0.8125 - } - }, - "auroc": 0.9660473958333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9805729166666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 752, - "fn": 48, - "accuracy": 0.94 - }, - "0.01": { - "tp": 691, - "fn": 109, - "accuracy": 0.86375 - } - }, - "auroc": 0.9733101562500001 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9759354166666666 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9812510416666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9785932291666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9784520833333332 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9813979166666666 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9799249999999999 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9771937500000001 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9813244791666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 785, - "fn": 15, - "accuracy": 0.98125 - } - }, - "auroc": 0.9792591145833334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 128, - "fn": 72, - "accuracy": 0.64 - }, - "0.01": { - "tp": 81, - "fn": 119, - "accuracy": 0.405 - } - }, - "auroc": 0.8952322916666667 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9871500000000001 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 323, - "fn": 77, - "accuracy": 0.8075 - }, - "0.01": { - "tp": 264, - "fn": 136, - "accuracy": 0.66 - } - }, - "auroc": 0.9411911458333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9856031250000001 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.983946875 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.984775 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 324, - "fn": 76, - "accuracy": 0.81 - }, - "0.01": { - "tp": 266, - "fn": 134, - "accuracy": 0.665 - } - }, - "auroc": 0.9404177083333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9855484375000001 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 713, - "fn": 87, - "accuracy": 0.89125 - }, - "0.01": { - "tp": 642, - "fn": 158, - "accuracy": 0.8025 - } - }, - "auroc": 0.9629830729166666 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - }, - "0.01": { - "tp": 116, - "fn": 84, - "accuracy": 0.58 - } - }, - "auroc": 0.93245625 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99370625 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - }, - "0.01": { - "tp": 316, - "fn": 84, - "accuracy": 0.79 - } - }, - "auroc": 0.9630812499999999 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9756010416666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9846822916666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9801416666666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - }, - "0.01": { - "tp": 297, - "fn": 103, - "accuracy": 0.7425 - } - }, - "auroc": 0.9540286458333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9891942708333332 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 756, - "fn": 44, - "accuracy": 0.945 - }, - "0.01": { - "tp": 687, - "fn": 113, - "accuracy": 0.85875 - } - }, - "auroc": 0.9716114583333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9757489583333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9728697916666667 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.974309375 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9748354166666666 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9752208333333334 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.975028125 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9752921875 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9740453125 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 787, - "fn": 13, - "accuracy": 0.98375 - } - }, - "auroc": 0.97466875 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9735916666666666 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9735916666666666 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.972471875 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.972471875 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9730317708333334 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9730317708333334 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9754197916666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9754197916666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9713729166666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9713729166666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9733963541666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9733963541666666 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9550875 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9550875 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.958340625 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.958340625 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9567140624999999 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9567140624999999 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9635 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9635 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9598875 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9598875 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.96169375 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.96169375 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.983296875 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.983296875 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9738114583333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9738114583333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9785541666666666 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9785541666666666 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2048, - "fn": 152, - "accuracy": 0.9309090909090909 - }, - "0.01": { - "tp": 1879, - "fn": 321, - "accuracy": 0.8540909090909091 - } - }, - "auroc": 0.9586353219696969 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1186, - "fn": 14, - "accuracy": 0.9883333333333333 - }, - "0.01": { - "tp": 1166, - "fn": 34, - "accuracy": 0.9716666666666667 - } - }, - "auroc": 0.9825626736111112 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3234, - "fn": 166, - "accuracy": 0.9511764705882353 - }, - "0.01": { - "tp": 3045, - "fn": 355, - "accuracy": 0.8955882352941177 - } - }, - "auroc": 0.9670802696078431 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2174, - "fn": 26, - "accuracy": 0.9881818181818182 - }, - "0.01": { - "tp": 2119, - "fn": 81, - "accuracy": 0.9631818181818181 - } - }, - "auroc": 0.9745882575757576 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1172, - "fn": 28, - "accuracy": 0.9766666666666667 - }, - "0.01": { - "tp": 1142, - "fn": 58, - "accuracy": 0.9516666666666667 - } - }, - "auroc": 0.9788661458333334 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3346, - "fn": 54, - "accuracy": 0.9841176470588235 - }, - "0.01": { - "tp": 3261, - "fn": 139, - "accuracy": 0.9591176470588235 - } - }, - "auroc": 0.9760981004901962 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 4222, - "fn": 178, - "accuracy": 0.9595454545454546 - }, - "0.01": { - "tp": 3998, - "fn": 402, - "accuracy": 0.9086363636363637 - } - }, - "auroc": 0.9666117897727272 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2358, - "fn": 42, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 2308, - "fn": 92, - "accuracy": 0.9616666666666667 - } - }, - "auroc": 0.9807144097222222 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 6580, - "fn": 220, - "accuracy": 0.9676470588235294 - }, - "0.01": { - "tp": 6306, - "fn": 494, - "accuracy": 0.9273529411764706 - } - }, - "auroc": 0.9715891850490196 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9796072916666666 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9760010416666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9778041666666666 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9759385416666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9713333333333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9736359375000001 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9777729166666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9736671875 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 776, - "fn": 24, - "accuracy": 0.97 - } - }, - "auroc": 0.9757200520833333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - }, - "0.01": { - "tp": 113, - "fn": 87, - "accuracy": 0.565 - } - }, - "auroc": 0.9282260416666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9832208333333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - }, - "0.01": { - "tp": 296, - "fn": 104, - "accuracy": 0.74 - } - }, - "auroc": 0.9557234375 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - } - }, - "auroc": 0.954784375 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9672020833333332 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 339, - "fn": 61, - "accuracy": 0.8475 - } - }, - "auroc": 0.9609932291666667 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - }, - "0.01": { - "tp": 280, - "fn": 120, - "accuracy": 0.7 - } - }, - "auroc": 0.9415052083333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - } - }, - "auroc": 0.9752114583333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 739, - "fn": 61, - "accuracy": 0.92375 - }, - "0.01": { - "tp": 635, - "fn": 165, - "accuracy": 0.79375 - } - }, - "auroc": 0.9583583333333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9773916666666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9764302083333333 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9769109375 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9745895833333333 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.9594979166666666 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.96704375 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.975990625 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9679640625 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 741, - "fn": 59, - "accuracy": 0.92625 - } - }, - "auroc": 0.9719773437499999 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 129, - "fn": 71, - "accuracy": 0.645 - }, - "0.01": { - "tp": 77, - "fn": 123, - "accuracy": 0.385 - } - }, - "auroc": 0.8926520833333332 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9803125 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 324, - "fn": 76, - "accuracy": 0.81 - }, - "0.01": { - "tp": 257, - "fn": 143, - "accuracy": 0.6425 - } - }, - "auroc": 0.9364822916666666 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 143, - "fn": 57, - "accuracy": 0.715 - } - }, - "auroc": 0.9415052083333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.972684375 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 325, - "fn": 75, - "accuracy": 0.8125 - } - }, - "auroc": 0.9570947916666668 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 320, - "fn": 80, - "accuracy": 0.8 - }, - "0.01": { - "tp": 220, - "fn": 180, - "accuracy": 0.55 - } - }, - "auroc": 0.9170786458333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - } - }, - "auroc": 0.9764984375000001 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 707, - "fn": 93, - "accuracy": 0.88375 - }, - "0.01": { - "tp": 582, - "fn": 218, - "accuracy": 0.7275 - } - }, - "auroc": 0.9467885416666668 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - }, - "0.01": { - "tp": 102, - "fn": 98, - "accuracy": 0.51 - } - }, - "auroc": 0.9190468749999999 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9837583333333333 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - }, - "0.01": { - "tp": 295, - "fn": 105, - "accuracy": 0.7375 - } - }, - "auroc": 0.9514026041666668 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - }, - "0.01": { - "tp": 89, - "fn": 111, - "accuracy": 0.445 - } - }, - "auroc": 0.8995989583333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - } - }, - "auroc": 0.9640291666666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 252, - "fn": 148, - "accuracy": 0.63 - } - }, - "auroc": 0.9318140625 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 335, - "fn": 65, - "accuracy": 0.8375 - }, - "0.01": { - "tp": 191, - "fn": 209, - "accuracy": 0.4775 - } - }, - "auroc": 0.9093229166666668 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.97389375 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 718, - "fn": 82, - "accuracy": 0.8975 - }, - "0.01": { - "tp": 547, - "fn": 253, - "accuracy": 0.68375 - } - }, - "auroc": 0.9416083333333334 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9820833333333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.98099375 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9815385416666667 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9775260416666667 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9688729166666668 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9731994791666667 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9798046875 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9749333333333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 775, - "fn": 25, - "accuracy": 0.96875 - } - }, - "auroc": 0.9773690104166667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.9471479166666666 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.9471479166666666 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 150, - "fn": 50, - "accuracy": 0.75 - } - }, - "auroc": 0.9440833333333334 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 150, - "fn": 50, - "accuracy": 0.75 - } - }, - "auroc": 0.9440833333333334 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 297, - "fn": 103, - "accuracy": 0.7425 - } - }, - "auroc": 0.945615625 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 297, - "fn": 103, - "accuracy": 0.7425 - } - }, - "auroc": 0.945615625 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - } - }, - "auroc": 0.9348614583333333 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - } - }, - "auroc": 0.9348614583333333 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - }, - "0.01": { - "tp": 107, - "fn": 93, - "accuracy": 0.535 - } - }, - "auroc": 0.9144385416666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - }, - "0.01": { - "tp": 107, - "fn": 93, - "accuracy": 0.535 - } - }, - "auroc": 0.9144385416666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - }, - "0.01": { - "tp": 237, - "fn": 163, - "accuracy": 0.5925 - } - }, - "auroc": 0.9246500000000001 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - }, - "0.01": { - "tp": 237, - "fn": 163, - "accuracy": 0.5925 - } - }, - "auroc": 0.9246500000000001 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.96193125 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.96193125 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9533385416666665 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9533385416666665 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9576348958333334 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9576348958333334 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.963084375 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.963084375 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 137, - "fn": 63, - "accuracy": 0.685 - } - }, - "auroc": 0.9381854166666665 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 137, - "fn": 63, - "accuracy": 0.685 - } - }, - "auroc": 0.9381854166666665 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 334, - "fn": 66, - "accuracy": 0.835 - } - }, - "auroc": 0.9506348958333334 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 334, - "fn": 66, - "accuracy": 0.835 - } - }, - "auroc": 0.9506348958333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.9668989583333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.9668989583333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 143, - "fn": 57, - "accuracy": 0.715 - } - }, - "auroc": 0.9431302083333333 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 143, - "fn": 57, - "accuracy": 0.715 - } - }, - "auroc": 0.9431302083333333 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 317, - "fn": 83, - "accuracy": 0.7925 - } - }, - "auroc": 0.9550145833333333 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 317, - "fn": 83, - "accuracy": 0.7925 - } - }, - "auroc": 0.9550145833333333 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2033, - "fn": 167, - "accuracy": 0.9240909090909091 - }, - "0.01": { - "tp": 1726, - "fn": 474, - "accuracy": 0.7845454545454545 - } - }, - "auroc": 0.9502664772727272 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1182, - "fn": 18, - "accuracy": 0.985 - }, - "0.01": { - "tp": 1143, - "fn": 57, - "accuracy": 0.9525 - } - }, - "auroc": 0.9801194444444444 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3215, - "fn": 185, - "accuracy": 0.9455882352941176 - }, - "0.01": { - "tp": 2869, - "fn": 531, - "accuracy": 0.8438235294117648 - } - }, - "auroc": 0.9608028186274511 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2094, - "fn": 106, - "accuracy": 0.9518181818181818 - }, - "0.01": { - "tp": 1710, - "fn": 490, - "accuracy": 0.7772727272727272 - } - }, - "auroc": 0.9470107954545455 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1155, - "fn": 45, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 1049, - "fn": 151, - "accuracy": 0.8741666666666666 - } - }, - "auroc": 0.9672699652777778 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3249, - "fn": 151, - "accuracy": 0.9555882352941176 - }, - "0.01": { - "tp": 2759, - "fn": 641, - "accuracy": 0.8114705882352942 - } - }, - "auroc": 0.9541610906862744 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 4127, - "fn": 273, - "accuracy": 0.9379545454545455 - }, - "0.01": { - "tp": 3436, - "fn": 964, - "accuracy": 0.7809090909090909 - } - }, - "auroc": 0.9486386363636364 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2337, - "fn": 63, - "accuracy": 0.97375 - }, - "0.01": { - "tp": 2192, - "fn": 208, - "accuracy": 0.9133333333333333 - } - }, - "auroc": 0.9736947048611111 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 6464, - "fn": 336, - "accuracy": 0.9505882352941176 - }, - "0.01": { - "tp": 5628, - "fn": 1172, - "accuracy": 0.8276470588235294 - } - }, - "auroc": 0.9574819546568628 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9861322916666666 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.980571875 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9833520833333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9830895833333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9746114583333334 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9788505208333332 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9846109375000001 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9775916666666666 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 789, - "fn": 11, - "accuracy": 0.98625 - } - }, - "auroc": 0.9811013020833333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - }, - "0.01": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - } - }, - "auroc": 0.93578125 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9873541666666668 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 308, - "fn": 92, - "accuracy": 0.77 - } - }, - "auroc": 0.9615677083333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9764999999999999 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.967865625 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - } - }, - "auroc": 0.9721828125 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 298, - "fn": 102, - "accuracy": 0.745 - } - }, - "auroc": 0.9561406250000001 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - } - }, - "auroc": 0.9776098958333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 743, - "fn": 57, - "accuracy": 0.92875 - }, - "0.01": { - "tp": 659, - "fn": 141, - "accuracy": 0.82375 - } - }, - "auroc": 0.9668752604166667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9840822916666666 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9766614583333333 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9803718749999999 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.980925 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9603916666666666 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 359, - "fn": 41, - "accuracy": 0.8975 - } - }, - "auroc": 0.9706583333333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9825036458333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9685265625 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 752, - "fn": 48, - "accuracy": 0.94 - } - }, - "auroc": 0.9755151041666666 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 132, - "fn": 68, - "accuracy": 0.66 - }, - "0.01": { - "tp": 85, - "fn": 115, - "accuracy": 0.425 - } - }, - "auroc": 0.8987729166666667 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9843760416666667 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 326, - "fn": 74, - "accuracy": 0.815 - }, - "0.01": { - "tp": 267, - "fn": 133, - "accuracy": 0.6675 - } - }, - "auroc": 0.9415744791666667 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 157, - "fn": 43, - "accuracy": 0.785 - } - }, - "auroc": 0.9596156249999999 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.973665625 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 339, - "fn": 61, - "accuracy": 0.8475 - } - }, - "auroc": 0.966640625 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 316, - "fn": 84, - "accuracy": 0.79 - }, - "0.01": { - "tp": 242, - "fn": 158, - "accuracy": 0.605 - } - }, - "auroc": 0.9291942708333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - } - }, - "auroc": 0.9790208333333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 702, - "fn": 98, - "accuracy": 0.8775 - }, - "0.01": { - "tp": 606, - "fn": 194, - "accuracy": 0.7575 - } - }, - "auroc": 0.9541075520833334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - }, - "0.01": { - "tp": 110, - "fn": 90, - "accuracy": 0.55 - } - }, - "auroc": 0.9245260416666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9882697916666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - }, - "0.01": { - "tp": 306, - "fn": 94, - "accuracy": 0.765 - } - }, - "auroc": 0.9563979166666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - }, - "0.01": { - "tp": 131, - "fn": 69, - "accuracy": 0.655 - } - }, - "auroc": 0.9235270833333333 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9675135416666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - }, - "0.01": { - "tp": 295, - "fn": 105, - "accuracy": 0.7375 - } - }, - "auroc": 0.9455203125000001 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 337, - "fn": 63, - "accuracy": 0.8425 - }, - "0.01": { - "tp": 241, - "fn": 159, - "accuracy": 0.6025 - } - }, - "auroc": 0.9240265625 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.9778916666666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 724, - "fn": 76, - "accuracy": 0.905 - }, - "0.01": { - "tp": 601, - "fn": 199, - "accuracy": 0.75125 - } - }, - "auroc": 0.9509591145833333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.987484375 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9818562500000001 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9846703124999999 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.983121875 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9721229166666667 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9776223958333332 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.985303125 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9769895833333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 781, - "fn": 19, - "accuracy": 0.97625 - } - }, - "auroc": 0.9811463541666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9591208333333333 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9591208333333333 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9554833333333334 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9554833333333334 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 329, - "fn": 71, - "accuracy": 0.8225 - } - }, - "auroc": 0.9573020833333332 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 329, - "fn": 71, - "accuracy": 0.8225 - } - }, - "auroc": 0.9573020833333332 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.9562479166666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.9562479166666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 140, - "fn": 60, - "accuracy": 0.7 - } - }, - "auroc": 0.9356802083333333 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 140, - "fn": 60, - "accuracy": 0.7 - } - }, - "auroc": 0.9356802083333333 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - }, - "0.01": { - "tp": 311, - "fn": 89, - "accuracy": 0.7775 - } - }, - "auroc": 0.9459640625000001 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - }, - "0.01": { - "tp": 311, - "fn": 89, - "accuracy": 0.7775 - } - }, - "auroc": 0.9459640625000001 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9655708333333334 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9655708333333334 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9590145833333333 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9590145833333333 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9622927083333334 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9622927083333334 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9762072916666666 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9762072916666666 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.956778125 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.956778125 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9664927083333332 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9664927083333332 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9792864583333333 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9792864583333333 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - } - }, - "auroc": 0.9564770833333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - } - }, - "auroc": 0.9564770833333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9678817708333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9678817708333334 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2036, - "fn": 164, - "accuracy": 0.9254545454545454 - }, - "0.01": { - "tp": 1835, - "fn": 365, - "accuracy": 0.8340909090909091 - } - }, - "auroc": 0.9593829545454545 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1180, - "fn": 20, - "accuracy": 0.9833333333333333 - }, - "0.01": { - "tp": 1158, - "fn": 42, - "accuracy": 0.965 - } - }, - "auroc": 0.9831815972222222 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3216, - "fn": 184, - "accuracy": 0.9458823529411765 - }, - "0.01": { - "tp": 2993, - "fn": 407, - "accuracy": 0.8802941176470588 - } - }, - "auroc": 0.967782475490196 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2100, - "fn": 100, - "accuracy": 0.9545454545454546 - }, - "0.01": { - "tp": 1912, - "fn": 288, - "accuracy": 0.8690909090909091 - } - }, - "auroc": 0.960928409090909 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1160, - "fn": 40, - "accuracy": 0.9666666666666667 - }, - "0.01": { - "tp": 1058, - "fn": 142, - "accuracy": 0.8816666666666667 - } - }, - "auroc": 0.9693618055555555 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3260, - "fn": 140, - "accuracy": 0.9588235294117647 - }, - "0.01": { - "tp": 2970, - "fn": 430, - "accuracy": 0.8735294117647059 - } - }, - "auroc": 0.9639049019607844 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 4136, - "fn": 264, - "accuracy": 0.94 - }, - "0.01": { - "tp": 3747, - "fn": 653, - "accuracy": 0.8515909090909091 - } - }, - "auroc": 0.9601556818181818 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2340, - "fn": 60, - "accuracy": 0.975 - }, - "0.01": { - "tp": 2216, - "fn": 184, - "accuracy": 0.9233333333333333 - } - }, - "auroc": 0.976271701388889 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 6476, - "fn": 324, - "accuracy": 0.9523529411764706 - }, - "0.01": { - "tp": 5963, - "fn": 837, - "accuracy": 0.8769117647058824 - } - }, - "auroc": 0.9658436887254903 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9866395833333332 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9805291666666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9835843750000001 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9838333333333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.974709375 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9792713541666667 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9852364583333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9776192708333333 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 789, - "fn": 11, - "accuracy": 0.98625 - } - }, - "auroc": 0.9814278645833334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - }, - "0.01": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - } - }, - "auroc": 0.9356895833333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9832208333333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 302, - "fn": 98, - "accuracy": 0.755 - } - }, - "auroc": 0.9594552083333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9706833333333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9672020833333332 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 344, - "fn": 56, - "accuracy": 0.86 - } - }, - "auroc": 0.9689427083333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 291, - "fn": 109, - "accuracy": 0.7275 - } - }, - "auroc": 0.9531864583333334 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - } - }, - "auroc": 0.9752114583333333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 741, - "fn": 59, - "accuracy": 0.92625 - }, - "0.01": { - "tp": 646, - "fn": 154, - "accuracy": 0.8075 - } - }, - "auroc": 0.9641989583333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9841520833333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9761249999999999 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9801385416666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9822458333333333 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 160, - "fn": 40, - "accuracy": 0.8 - } - }, - "auroc": 0.9591822916666667 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9707140624999999 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9831989583333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9676536458333334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 748, - "fn": 52, - "accuracy": 0.935 - } - }, - "auroc": 0.9754263020833334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 132, - "fn": 68, - "accuracy": 0.66 - }, - "0.01": { - "tp": 85, - "fn": 115, - "accuracy": 0.425 - } - }, - "auroc": 0.8987333333333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.98414375 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 327, - "fn": 73, - "accuracy": 0.8175 - }, - "0.01": { - "tp": 266, - "fn": 134, - "accuracy": 0.665 - } - }, - "auroc": 0.9414385416666666 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.9512427083333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9665447916666667 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 324, - "fn": 76, - "accuracy": 0.81 - } - }, - "auroc": 0.9588937500000001 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 314, - "fn": 86, - "accuracy": 0.785 - }, - "0.01": { - "tp": 232, - "fn": 168, - "accuracy": 0.58 - } - }, - "auroc": 0.9249880208333333 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9753442708333334 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 700, - "fn": 100, - "accuracy": 0.875 - }, - "0.01": { - "tp": 590, - "fn": 210, - "accuracy": 0.7375 - } - }, - "auroc": 0.9501661458333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - }, - "0.01": { - "tp": 111, - "fn": 89, - "accuracy": 0.555 - } - }, - "auroc": 0.9250666666666666 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9865312500000001 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 306, - "fn": 94, - "accuracy": 0.765 - } - }, - "auroc": 0.9557989583333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - }, - "0.01": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - } - }, - "auroc": 0.913596875 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 159, - "fn": 41, - "accuracy": 0.795 - } - }, - "auroc": 0.9619197916666666 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - }, - "0.01": { - "tp": 278, - "fn": 122, - "accuracy": 0.695 - } - }, - "auroc": 0.9377583333333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - }, - "0.01": { - "tp": 230, - "fn": 170, - "accuracy": 0.575 - } - }, - "auroc": 0.9193317708333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9742255208333334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 715, - "fn": 85, - "accuracy": 0.89375 - }, - "0.01": { - "tp": 584, - "fn": 216, - "accuracy": 0.73 - } - }, - "auroc": 0.9467786458333334 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9877218750000001 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9817979166666666 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9847598958333332 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9834010416666668 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.970953125 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9771770833333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9855614583333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9763755208333333 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 780, - "fn": 20, - "accuracy": 0.975 - } - }, - "auroc": 0.9809684895833334 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9586385416666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9586385416666667 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9531302083333333 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9531302083333333 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - } - }, - "auroc": 0.9558843750000001 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - } - }, - "auroc": 0.9558843750000001 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9538979166666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9538979166666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - } - }, - "auroc": 0.9304760416666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - } - }, - "auroc": 0.9304760416666666 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - }, - "0.01": { - "tp": 299, - "fn": 101, - "accuracy": 0.7475 - } - }, - "auroc": 0.9421869791666667 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - }, - "0.01": { - "tp": 299, - "fn": 101, - "accuracy": 0.7475 - } - }, - "auroc": 0.9421869791666667 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9659385416666668 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9659385416666668 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9590291666666666 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9590291666666666 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9624838541666667 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9624838541666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9771541666666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9771541666666667 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9526895833333333 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9526895833333333 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9649218749999999 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9649218749999999 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9791541666666667 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9791541666666667 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9551802083333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9551802083333334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9671671875000001 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9671671875000001 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2037, - "fn": 163, - "accuracy": 0.9259090909090909 - }, - "0.01": { - "tp": 1833, - "fn": 367, - "accuracy": 0.8331818181818181 - } - }, - "auroc": 0.9593442234848484 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1181, - "fn": 19, - "accuracy": 0.9841666666666666 - }, - "0.01": { - "tp": 1148, - "fn": 52, - "accuracy": 0.9566666666666667 - } - }, - "auroc": 0.9820579861111111 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3218, - "fn": 182, - "accuracy": 0.9464705882352941 - }, - "0.01": { - "tp": 2981, - "fn": 419, - "accuracy": 0.8767647058823529 - } - }, - "auroc": 0.9673608455882354 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2086, - "fn": 114, - "accuracy": 0.9481818181818182 - }, - "0.01": { - "tp": 1866, - "fn": 334, - "accuracy": 0.8481818181818181 - } - }, - "auroc": 0.9577734848484849 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1154, - "fn": 46, - "accuracy": 0.9616666666666667 - }, - "0.01": { - "tp": 1045, - "fn": 155, - "accuracy": 0.8708333333333333 - } - }, - "auroc": 0.9667519097222222 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3240, - "fn": 160, - "accuracy": 0.9529411764705882 - }, - "0.01": { - "tp": 2911, - "fn": 489, - "accuracy": 0.8561764705882353 - } - }, - "auroc": 0.9609423406862745 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 4123, - "fn": 277, - "accuracy": 0.9370454545454545 - }, - "0.01": { - "tp": 3699, - "fn": 701, - "accuracy": 0.8406818181818182 - } - }, - "auroc": 0.9585588541666666 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2335, - "fn": 65, - "accuracy": 0.9729166666666667 - }, - "0.01": { - "tp": 2193, - "fn": 207, - "accuracy": 0.91375 - } - }, - "auroc": 0.9744049479166668 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 6458, - "fn": 342, - "accuracy": 0.9497058823529412 - }, - "0.01": { - "tp": 5892, - "fn": 908, - "accuracy": 0.8664705882352941 - } - }, - "auroc": 0.9641515931372548 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2385, - "fn": 15, - "accuracy": 0.99375 - } - }, - "auroc": 0.9858024305555555 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2374, - "fn": 26, - "accuracy": 0.9891666666666666 - } - }, - "auroc": 0.9805078993055556 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4759, - "fn": 41, - "accuracy": 0.9914583333333333 - } - }, - "auroc": 0.9831551649305554 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 2362, - "fn": 38, - "accuracy": 0.9841666666666666 - } - }, - "auroc": 0.9835625868055555 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 2324, - "fn": 76, - "accuracy": 0.9683333333333334 - } - }, - "auroc": 0.9752375868055556 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4796, - "fn": 4, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 4686, - "fn": 114, - "accuracy": 0.97625 - } - }, - "auroc": 0.9794000868055555 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4798, - "fn": 2, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 4747, - "fn": 53, - "accuracy": 0.9889583333333334 - } - }, - "auroc": 0.9846825086805555 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4798, - "fn": 2, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 4698, - "fn": 102, - "accuracy": 0.97875 - } - }, - "auroc": 0.9778727430555556 - }, - { - "domain": "poetry", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9596, - "fn": 4, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 9445, - "fn": 155, - "accuracy": 0.9838541666666667 - } - }, - "auroc": 0.9812776258680556 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2037, - "fn": 363, - "accuracy": 0.84875 - }, - "0.01": { - "tp": 1432, - "fn": 968, - "accuracy": 0.5966666666666667 - } - }, - "auroc": 0.934275173611111 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2307, - "fn": 93, - "accuracy": 0.96125 - }, - "0.01": { - "tp": 2146, - "fn": 254, - "accuracy": 0.8941666666666667 - } - }, - "auroc": 0.9791703993055555 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4344, - "fn": 456, - "accuracy": 0.905 - }, - "0.01": { - "tp": 3578, - "fn": 1222, - "accuracy": 0.7454166666666666 - } - }, - "auroc": 0.9567227864583333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2321, - "fn": 79, - "accuracy": 0.9670833333333333 - }, - "0.01": { - "tp": 2107, - "fn": 293, - "accuracy": 0.8779166666666667 - } - }, - "auroc": 0.9721481770833333 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2176, - "fn": 224, - "accuracy": 0.9066666666666666 - }, - "0.01": { - "tp": 1994, - "fn": 406, - "accuracy": 0.8308333333333333 - } - }, - "auroc": 0.9618491319444444 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4497, - "fn": 303, - "accuracy": 0.936875 - }, - "0.01": { - "tp": 4101, - "fn": 699, - "accuracy": 0.854375 - } - }, - "auroc": 0.9669986545138889 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4358, - "fn": 442, - "accuracy": 0.9079166666666667 - }, - "0.01": { - "tp": 3539, - "fn": 1261, - "accuracy": 0.7372916666666667 - } - }, - "auroc": 0.9532116753472223 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4483, - "fn": 317, - "accuracy": 0.9339583333333333 - }, - "0.01": { - "tp": 4140, - "fn": 660, - "accuracy": 0.8625 - } - }, - "auroc": 0.970509765625 - }, - { - "domain": "poetry", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 8841, - "fn": 759, - "accuracy": 0.9209375 - }, - "0.01": { - "tp": 7679, - "fn": 1921, - "accuracy": 0.7998958333333334 - } - }, - "auroc": 0.9618607204861112 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2388, - "fn": 12, - "accuracy": 0.995 - }, - "0.01": { - "tp": 2380, - "fn": 20, - "accuracy": 0.9916666666666667 - } - }, - "auroc": 0.9839497395833333 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2356, - "fn": 44, - "accuracy": 0.9816666666666667 - }, - "0.01": { - "tp": 2290, - "fn": 110, - "accuracy": 0.9541666666666667 - } - }, - "auroc": 0.9758147569444444 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4744, - "fn": 56, - "accuracy": 0.9883333333333333 - }, - "0.01": { - "tp": 4670, - "fn": 130, - "accuracy": 0.9729166666666667 - } - }, - "auroc": 0.9798822482638889 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 2349, - "fn": 51, - "accuracy": 0.97875 - } - }, - "auroc": 0.9818972222222222 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2330, - "fn": 70, - "accuracy": 0.9708333333333333 - }, - "0.01": { - "tp": 1929, - "fn": 471, - "accuracy": 0.80375 - } - }, - "auroc": 0.9592526909722222 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4726, - "fn": 74, - "accuracy": 0.9845833333333334 - }, - "0.01": { - "tp": 4278, - "fn": 522, - "accuracy": 0.89125 - } - }, - "auroc": 0.9705749565972221 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4784, - "fn": 16, - "accuracy": 0.9966666666666667 - }, - "0.01": { - "tp": 4729, - "fn": 71, - "accuracy": 0.9852083333333334 - } - }, - "auroc": 0.9829234809027778 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4686, - "fn": 114, - "accuracy": 0.97625 - }, - "0.01": { - "tp": 4219, - "fn": 581, - "accuracy": 0.8789583333333333 - } - }, - "auroc": 0.9675337239583334 - }, - { - "domain": "poetry", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9470, - "fn": 130, - "accuracy": 0.9864583333333333 - }, - "0.01": { - "tp": 8948, - "fn": 652, - "accuracy": 0.9320833333333334 - } - }, - "auroc": 0.9752286024305555 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 1585, - "fn": 815, - "accuracy": 0.6604166666666667 - }, - "0.01": { - "tp": 1006, - "fn": 1394, - "accuracy": 0.4191666666666667 - } - }, - "auroc": 0.8968890625000001 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2337, - "fn": 63, - "accuracy": 0.97375 - }, - "0.01": { - "tp": 2180, - "fn": 220, - "accuracy": 0.9083333333333333 - } - }, - "auroc": 0.983679513888889 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 3922, - "fn": 878, - "accuracy": 0.8170833333333334 - }, - "0.01": { - "tp": 3186, - "fn": 1614, - "accuracy": 0.66375 - } - }, - "auroc": 0.9402842881944443 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2227, - "fn": 173, - "accuracy": 0.9279166666666666 - }, - "0.01": { - "tp": 1836, - "fn": 564, - "accuracy": 0.765 - } - }, - "auroc": 0.9556590277777779 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2292, - "fn": 108, - "accuracy": 0.955 - }, - "0.01": { - "tp": 2093, - "fn": 307, - "accuracy": 0.8720833333333333 - } - }, - "auroc": 0.9669627604166665 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4519, - "fn": 281, - "accuracy": 0.9414583333333333 - }, - "0.01": { - "tp": 3929, - "fn": 871, - "accuracy": 0.8185416666666666 - } - }, - "auroc": 0.9613108940972223 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 3812, - "fn": 988, - "accuracy": 0.7941666666666667 - }, - "0.01": { - "tp": 2842, - "fn": 1958, - "accuracy": 0.5920833333333333 - } - }, - "auroc": 0.9262740451388889 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4629, - "fn": 171, - "accuracy": 0.964375 - }, - "0.01": { - "tp": 4273, - "fn": 527, - "accuracy": 0.8902083333333334 - } - }, - "auroc": 0.9753211371527779 - }, - { - "domain": "poetry", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 8441, - "fn": 1159, - "accuracy": 0.8792708333333333 - }, - "0.01": { - "tp": 7115, - "fn": 2485, - "accuracy": 0.7411458333333333 - } - }, - "auroc": 0.9507975911458334 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 1991, - "fn": 409, - "accuracy": 0.8295833333333333 - }, - "0.01": { - "tp": 1334, - "fn": 1066, - "accuracy": 0.5558333333333333 - } - }, - "auroc": 0.92482109375 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2358, - "fn": 42, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 2298, - "fn": 102, - "accuracy": 0.9575 - } - }, - "auroc": 0.9846855902777778 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4349, - "fn": 451, - "accuracy": 0.9060416666666666 - }, - "0.01": { - "tp": 3632, - "fn": 1168, - "accuracy": 0.7566666666666667 - } - }, - "auroc": 0.9547533420138888 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2072, - "fn": 328, - "accuracy": 0.8633333333333333 - }, - "0.01": { - "tp": 1541, - "fn": 859, - "accuracy": 0.6420833333333333 - } - }, - "auroc": 0.9239988715277778 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2237, - "fn": 163, - "accuracy": 0.9320833333333334 - }, - "0.01": { - "tp": 1903, - "fn": 497, - "accuracy": 0.7929166666666667 - } - }, - "auroc": 0.9607776041666667 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4309, - "fn": 491, - "accuracy": 0.8977083333333333 - }, - "0.01": { - "tp": 3444, - "fn": 1356, - "accuracy": 0.7175 - } - }, - "auroc": 0.9423882378472224 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4063, - "fn": 737, - "accuracy": 0.8464583333333333 - }, - "0.01": { - "tp": 2875, - "fn": 1925, - "accuracy": 0.5989583333333334 - } - }, - "auroc": 0.9244099826388888 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4595, - "fn": 205, - "accuracy": 0.9572916666666667 - }, - "0.01": { - "tp": 4201, - "fn": 599, - "accuracy": 0.8752083333333334 - } - }, - "auroc": 0.9727315972222222 - }, - { - "domain": "poetry", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 8658, - "fn": 942, - "accuracy": 0.901875 - }, - "0.01": { - "tp": 7076, - "fn": 2524, - "accuracy": 0.7370833333333333 - } - }, - "auroc": 0.9485707899305555 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2390, - "fn": 10, - "accuracy": 0.9958333333333333 - } - }, - "auroc": 0.9870831597222223 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2386, - "fn": 14, - "accuracy": 0.9941666666666666 - }, - "0.01": { - "tp": 2372, - "fn": 28, - "accuracy": 0.9883333333333333 - } - }, - "auroc": 0.9816341145833334 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4786, - "fn": 14, - "accuracy": 0.9970833333333333 - }, - "0.01": { - "tp": 4762, - "fn": 38, - "accuracy": 0.9920833333333333 - } - }, - "auroc": 0.9843586371527777 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2388, - "fn": 12, - "accuracy": 0.995 - }, - "0.01": { - "tp": 2377, - "fn": 23, - "accuracy": 0.9904166666666666 - } - }, - "auroc": 0.9828390625000001 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2354, - "fn": 46, - "accuracy": 0.9808333333333333 - }, - "0.01": { - "tp": 2188, - "fn": 212, - "accuracy": 0.9116666666666666 - } - }, - "auroc": 0.9707521701388888 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4742, - "fn": 58, - "accuracy": 0.9879166666666667 - }, - "0.01": { - "tp": 4565, - "fn": 235, - "accuracy": 0.9510416666666667 - } - }, - "auroc": 0.9767956163194444 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4788, - "fn": 12, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 4767, - "fn": 33, - "accuracy": 0.993125 - } - }, - "auroc": 0.9849611111111111 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4740, - "fn": 60, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 4560, - "fn": 240, - "accuracy": 0.95 - } - }, - "auroc": 0.9761931423611112 - }, - { - "domain": "poetry", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9528, - "fn": 72, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 9327, - "fn": 273, - "accuracy": 0.9715625 - } - }, - "auroc": 0.9805771267361111 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2240, - "fn": 160, - "accuracy": 0.9333333333333333 - }, - "0.01": { - "tp": 1938, - "fn": 462, - "accuracy": 0.8075 - } - }, - "auroc": 0.956974392361111 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2240, - "fn": 160, - "accuracy": 0.9333333333333333 - }, - "0.01": { - "tp": 1938, - "fn": 462, - "accuracy": 0.8075 - } - }, - "auroc": 0.956974392361111 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2218, - "fn": 182, - "accuracy": 0.9241666666666667 - }, - "0.01": { - "tp": 1927, - "fn": 473, - "accuracy": 0.8029166666666666 - } - }, - "auroc": 0.9523285590277778 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2218, - "fn": 182, - "accuracy": 0.9241666666666667 - }, - "0.01": { - "tp": 1927, - "fn": 473, - "accuracy": 0.8029166666666666 - } - }, - "auroc": 0.9523285590277778 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4458, - "fn": 342, - "accuracy": 0.92875 - }, - "0.01": { - "tp": 3865, - "fn": 935, - "accuracy": 0.8052083333333333 - } - }, - "auroc": 0.9546514756944444 - }, - { - "domain": "poetry", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4458, - "fn": 342, - "accuracy": 0.92875 - }, - "0.01": { - "tp": 3865, - "fn": 935, - "accuracy": 0.8052083333333333 - } - }, - "auroc": 0.9546514756944444 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2266, - "fn": 134, - "accuracy": 0.9441666666666667 - }, - "0.01": { - "tp": 2005, - "fn": 395, - "accuracy": 0.8354166666666667 - } - }, - "auroc": 0.9552629340277778 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2266, - "fn": 134, - "accuracy": 0.9441666666666667 - }, - "0.01": { - "tp": 2005, - "fn": 395, - "accuracy": 0.8354166666666667 - } - }, - "auroc": 0.9552629340277778 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2169, - "fn": 231, - "accuracy": 0.90375 - }, - "0.01": { - "tp": 1634, - "fn": 766, - "accuracy": 0.6808333333333333 - } - }, - "auroc": 0.9347544270833335 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2169, - "fn": 231, - "accuracy": 0.90375 - }, - "0.01": { - "tp": 1634, - "fn": 766, - "accuracy": 0.6808333333333333 - } - }, - "auroc": 0.9347544270833335 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4435, - "fn": 365, - "accuracy": 0.9239583333333333 - }, - "0.01": { - "tp": 3639, - "fn": 1161, - "accuracy": 0.758125 - } - }, - "auroc": 0.9450086805555555 - }, - { - "domain": "poetry", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4435, - "fn": 365, - "accuracy": 0.9239583333333333 - }, - "0.01": { - "tp": 3639, - "fn": 1161, - "accuracy": 0.758125 - } - }, - "auroc": 0.9450086805555555 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2367, - "fn": 33, - "accuracy": 0.98625 - }, - "0.01": { - "tp": 2355, - "fn": 45, - "accuracy": 0.98125 - } - }, - "auroc": 0.9679644965277778 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2367, - "fn": 33, - "accuracy": 0.98625 - }, - "0.01": { - "tp": 2355, - "fn": 45, - "accuracy": 0.98125 - } - }, - "auroc": 0.9679644965277778 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2356, - "fn": 44, - "accuracy": 0.9816666666666667 - }, - "0.01": { - "tp": 2333, - "fn": 67, - "accuracy": 0.9720833333333333 - } - }, - "auroc": 0.9627736979166667 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2356, - "fn": 44, - "accuracy": 0.9816666666666667 - }, - "0.01": { - "tp": 2333, - "fn": 67, - "accuracy": 0.9720833333333333 - } - }, - "auroc": 0.9627736979166667 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4723, - "fn": 77, - "accuracy": 0.9839583333333334 - }, - "0.01": { - "tp": 4688, - "fn": 112, - "accuracy": 0.9766666666666667 - } - }, - "auroc": 0.9653690972222222 - }, - { - "domain": "poetry", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4723, - "fn": 77, - "accuracy": 0.9839583333333334 - }, - "0.01": { - "tp": 4688, - "fn": 112, - "accuracy": 0.9766666666666667 - } - }, - "auroc": 0.9653690972222222 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9778547743055556 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9778547743055556 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2390, - "fn": 10, - "accuracy": 0.9958333333333333 - }, - "0.01": { - "tp": 2197, - "fn": 203, - "accuracy": 0.9154166666666667 - } - }, - "auroc": 0.957038454861111 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2390, - "fn": 10, - "accuracy": 0.9958333333333333 - }, - "0.01": { - "tp": 2197, - "fn": 203, - "accuracy": 0.9154166666666667 - } - }, - "auroc": 0.957038454861111 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4790, - "fn": 10, - "accuracy": 0.9979166666666667 - }, - "0.01": { - "tp": 4593, - "fn": 207, - "accuracy": 0.956875 - } - }, - "auroc": 0.9674466145833333 - }, - { - "domain": "poetry", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4790, - "fn": 10, - "accuracy": 0.9979166666666667 - }, - "0.01": { - "tp": 4593, - "fn": 207, - "accuracy": 0.956875 - } - }, - "auroc": 0.9674466145833333 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2369, - "fn": 31, - "accuracy": 0.9870833333333333 - }, - "0.01": { - "tp": 2248, - "fn": 152, - "accuracy": 0.9366666666666666 - } - }, - "auroc": 0.9777164930555555 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2369, - "fn": 31, - "accuracy": 0.9870833333333333 - }, - "0.01": { - "tp": 2248, - "fn": 152, - "accuracy": 0.9366666666666666 - } - }, - "auroc": 0.9777164930555555 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2260, - "fn": 140, - "accuracy": 0.9416666666666667 - }, - "0.01": { - "tp": 1939, - "fn": 461, - "accuracy": 0.8079166666666666 - } - }, - "auroc": 0.9561778645833334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2260, - "fn": 140, - "accuracy": 0.9416666666666667 - }, - "0.01": { - "tp": 1939, - "fn": 461, - "accuracy": 0.8079166666666666 - } - }, - "auroc": 0.9561778645833334 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4629, - "fn": 171, - "accuracy": 0.964375 - }, - "0.01": { - "tp": 4187, - "fn": 613, - "accuracy": 0.8722916666666667 - } - }, - "auroc": 0.9669471788194444 - }, - { - "domain": "poetry", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4629, - "fn": 171, - "accuracy": 0.964375 - }, - "0.01": { - "tp": 4187, - "fn": 613, - "accuracy": 0.8722916666666667 - } - }, - "auroc": 0.9669471788194444 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 24443, - "fn": 1957, - "accuracy": 0.9258712121212122 - }, - "0.01": { - "tp": 21869, - "fn": 4531, - "accuracy": 0.8283712121212121 - } - }, - "auroc": 0.9589630681818182 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 14144, - "fn": 256, - "accuracy": 0.9822222222222222 - }, - "0.01": { - "tp": 13660, - "fn": 740, - "accuracy": 0.9486111111111111 - } - }, - "auroc": 0.980915379050926 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 38587, - "fn": 2213, - "accuracy": 0.9457598039215687 - }, - "0.01": { - "tp": 35529, - "fn": 5271, - "accuracy": 0.8708088235294118 - } - }, - "auroc": 0.966710942606209 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 25195, - "fn": 1205, - "accuracy": 0.9543560606060606 - }, - "0.01": { - "tp": 22602, - "fn": 3798, - "accuracy": 0.8561363636363636 - } - }, - "auroc": 0.9602889046717172 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 13787, - "fn": 613, - "accuracy": 0.9574305555555556 - }, - "0.01": { - "tp": 12431, - "fn": 1969, - "accuracy": 0.8632638888888889 - } - }, - "auroc": 0.965805324074074 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 38982, - "fn": 1818, - "accuracy": 0.9554411764705882 - }, - "0.01": { - "tp": 35033, - "fn": 5767, - "accuracy": 0.8586519607843137 - } - }, - "auroc": 0.9622358762254901 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 49638, - "fn": 3162, - "accuracy": 0.9401136363636363 - }, - "0.01": { - "tp": 44471, - "fn": 8329, - "accuracy": 0.8422537878787879 - } - }, - "auroc": 0.9596259864267678 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 27931, - "fn": 869, - "accuracy": 0.9698263888888888 - }, - "0.01": { - "tp": 26091, - "fn": 2709, - "accuracy": 0.9059375 - } - }, - "auroc": 0.9733603515625 - }, - { - "domain": "poetry", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 77569, - "fn": 4031, - "accuracy": 0.9506004901960784 - }, - "0.01": { - "tp": 70562, - "fn": 11038, - "accuracy": 0.8647303921568628 - } - }, - "auroc": 0.9644734094158496 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998552083333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996958333333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997755208333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995947916666666 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996473958333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997250000000001 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996979166666666 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997114583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9931989583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9907489583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9919739583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.971353125 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9802375000000001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9757953125000001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9822760416666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9854932291666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 785, - "fn": 15, - "accuracy": 0.98125 - } - }, - "auroc": 0.9838846354166667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995531249999999 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9906635416666666 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9951083333333334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995177083333334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9804197916666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9899687500000001 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995354166666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9855416666666668 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9925385416666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.995271875 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9837218750000001 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9894968749999999 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9466041666666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9528583333333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.94973125 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9709380208333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9682901041666666 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 760, - "fn": 40, - "accuracy": 0.95 - } - }, - "auroc": 0.9696140625 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981187499999999 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99576875 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99694375 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9558375000000001 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9547291666666666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9552833333333334 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.976978125 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9752489583333334 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - } - }, - "auroc": 0.9761135416666669 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997322916666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996989583333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999715625 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997083333333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981416666666666 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998925 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997203125 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989203125 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993203125 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974645833333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974645833333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9917427083333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9917427083333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9946036458333334 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9946036458333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9910645833333335 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9910645833333335 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9911687499999999 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9911687499999999 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9911166666666666 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9911166666666666 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999740625 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999740625 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997125 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997125 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997265625 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997265625 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999615625 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999615625 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995770833333334 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995770833333334 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995963541666666 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995963541666666 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974927083333334 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974927083333334 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.996684375 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.996684375 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9970885416666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9970885416666667 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2198, - "fn": 2, - "accuracy": 0.9990909090909091 - } - }, - "auroc": 0.9973734848484848 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1191, - "fn": 9, - "accuracy": 0.9925 - } - }, - "auroc": 0.9933829861111111 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3389, - "fn": 11, - "accuracy": 0.9967647058823529 - } - }, - "auroc": 0.9959650735294117 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2183, - "fn": 17, - "accuracy": 0.9922727272727273 - }, - "0.01": { - "tp": 2143, - "fn": 57, - "accuracy": 0.9740909090909091 - } - }, - "auroc": 0.9865000946969698 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1195, - "fn": 5, - "accuracy": 0.9958333333333333 - }, - "0.01": { - "tp": 1172, - "fn": 28, - "accuracy": 0.9766666666666667 - } - }, - "auroc": 0.9776810763888888 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3378, - "fn": 22, - "accuracy": 0.9935294117647059 - }, - "0.01": { - "tp": 3315, - "fn": 85, - "accuracy": 0.975 - } - }, - "auroc": 0.9833874999999999 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 4383, - "fn": 17, - "accuracy": 0.9961363636363636 - }, - "0.01": { - "tp": 4341, - "fn": 59, - "accuracy": 0.9865909090909091 - } - }, - "auroc": 0.9919367897727271 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2395, - "fn": 5, - "accuracy": 0.9979166666666667 - }, - "0.01": { - "tp": 2363, - "fn": 37, - "accuracy": 0.9845833333333334 - } - }, - "auroc": 0.98553203125 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 6778, - "fn": 22, - "accuracy": 0.9967647058823529 - }, - "0.01": { - "tp": 6704, - "fn": 96, - "accuracy": 0.9858823529411764 - } - }, - "auroc": 0.9896762867647059 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998552083333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996958333333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997755208333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995947916666666 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996473958333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997250000000001 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996979166666666 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997114583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9931989583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9907489583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9919739583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.971353125 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9802375000000001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9757953125000001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9822760416666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9854932291666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 785, - "fn": 15, - "accuracy": 0.98125 - } - }, - "auroc": 0.9838846354166667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995531249999999 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9906635416666666 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9951083333333334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995177083333334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9804197916666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9899687500000001 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995354166666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9855416666666668 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9925385416666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.995271875 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9837218750000001 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9894968749999999 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9466041666666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9528583333333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.94973125 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9709380208333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9682901041666666 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 760, - "fn": 40, - "accuracy": 0.95 - } - }, - "auroc": 0.9696140625 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981187499999999 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99576875 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99694375 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9558375000000001 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9547291666666666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9552833333333334 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.976978125 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9752489583333334 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - } - }, - "auroc": 0.9761135416666669 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997322916666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996989583333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999715625 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997083333333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981416666666666 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998925 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997203125 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989203125 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993203125 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974645833333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974645833333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9917427083333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9917427083333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9946036458333334 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9946036458333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9910645833333335 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9910645833333335 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9911687499999999 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9911687499999999 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9911166666666666 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9911166666666666 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999740625 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999740625 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997125 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997125 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997265625 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997265625 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999615625 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999615625 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995770833333334 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995770833333334 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995963541666666 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995963541666666 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974927083333334 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974927083333334 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.996684375 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.996684375 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9970885416666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9970885416666667 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2198, - "fn": 2, - "accuracy": 0.9990909090909091 - } - }, - "auroc": 0.9973734848484848 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1191, - "fn": 9, - "accuracy": 0.9925 - } - }, - "auroc": 0.9933829861111111 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3389, - "fn": 11, - "accuracy": 0.9967647058823529 - } - }, - "auroc": 0.9959650735294117 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2183, - "fn": 17, - "accuracy": 0.9922727272727273 - }, - "0.01": { - "tp": 2143, - "fn": 57, - "accuracy": 0.9740909090909091 - } - }, - "auroc": 0.9865000946969698 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1195, - "fn": 5, - "accuracy": 0.9958333333333333 - }, - "0.01": { - "tp": 1172, - "fn": 28, - "accuracy": 0.9766666666666667 - } - }, - "auroc": 0.9776810763888888 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3378, - "fn": 22, - "accuracy": 0.9935294117647059 - }, - "0.01": { - "tp": 3315, - "fn": 85, - "accuracy": 0.975 - } - }, - "auroc": 0.9833874999999999 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 4383, - "fn": 17, - "accuracy": 0.9961363636363636 - }, - "0.01": { - "tp": 4341, - "fn": 59, - "accuracy": 0.9865909090909091 - } - }, - "auroc": 0.9919367897727271 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2395, - "fn": 5, - "accuracy": 0.9979166666666667 - }, - "0.01": { - "tp": 2363, - "fn": 37, - "accuracy": 0.9845833333333334 - } - }, - "auroc": 0.98553203125 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 6778, - "fn": 22, - "accuracy": 0.9967647058823529 - }, - "0.01": { - "tp": 6704, - "fn": 96, - "accuracy": 0.9858823529411764 - } - }, - "auroc": 0.9896762867647059 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99999375 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997666666666667 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998802083333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997395833333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999675 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997072916666667 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998666666666667 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997208333333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99979375 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9956958333333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9897875 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9927416666666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.99176875 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9784375 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9851031250000001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9937322916666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9841125000000001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - } - }, - "auroc": 0.9889223958333333 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999510416666666 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9888010416666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9943760416666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999614583333334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9787479166666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9893546875 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99995625 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9837744791666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9918653645833333 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9967708333333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9906 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9936854166666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9825875000000001 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9753833333333333 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9789854166666666 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9896791666666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9829916666666666 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - } - }, - "auroc": 0.9863354166666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990270833333333 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9991958333333333 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9991114583333334 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9808375 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9688499999999999 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.97484375 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9899322916666666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9840229166666666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 789, - "fn": 11, - "accuracy": 0.98625 - } - }, - "auroc": 0.9869776041666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999729166666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999895833333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99998125 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999645833333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989645833333334 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994645833333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999687500000001 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994770833333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997229166666667 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9984354166666667 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9984354166666667 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9949802083333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9949802083333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9967078125000001 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9967078125000001 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9932812500000001 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9932812500000001 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.994196875 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.994196875 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9937390625000001 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9937390625000001 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999833333333333 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999833333333333 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999946875 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999946875 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999651041666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999651041666667 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99999375 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99999375 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999885416666666 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999885416666666 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999911458333333 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999911458333333 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.998384375 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.998384375 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985031249999999 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985031249999999 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.99844375 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.99844375 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2198, - "fn": 2, - "accuracy": 0.9990909090909091 - } - }, - "auroc": 0.9983172348484849 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1196, - "fn": 4, - "accuracy": 0.9966666666666667 - } - }, - "auroc": 0.9946901041666667 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3394, - "fn": 6, - "accuracy": 0.9982352941176471 - } - }, - "auroc": 0.9970370710784313 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2192, - "fn": 8, - "accuracy": 0.9963636363636363 - } - }, - "auroc": 0.9947704545454545 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1181, - "fn": 19, - "accuracy": 0.9841666666666666 - } - }, - "auroc": 0.9833430555555556 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3373, - "fn": 27, - "accuracy": 0.9920588235294118 - } - }, - "auroc": 0.9907372549019607 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 4400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4390, - "fn": 10, - "accuracy": 0.9977272727272727 - } - }, - "auroc": 0.9965438446969697 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2377, - "fn": 23, - "accuracy": 0.9904166666666666 - } - }, - "auroc": 0.989016579861111 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 6800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 6767, - "fn": 33, - "accuracy": 0.9951470588235294 - } - }, - "auroc": 0.9938871629901961 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999395833333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996770833333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998083333333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996614583333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994385416666667 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995499999999999 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998005208333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995578125 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996791666666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9919979166666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9880156250000001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9900067708333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9774854166666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9765020833333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.97699375 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9847416666666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9822588541666666 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 783, - "fn": 17, - "accuracy": 0.97875 - } - }, - "auroc": 0.9835002604166667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997885416666666 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.986690625 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9932395833333334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997427083333333 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9738145833333334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9867786458333333 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999765625 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9802526041666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9900091145833333 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9946197916666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9849208333333335 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9897703124999999 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9484385416666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.956046875 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9522427083333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9715291666666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9704838541666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 769, - "fn": 31, - "accuracy": 0.96125 - } - }, - "auroc": 0.9710065104166667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998296875 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9938750000000001 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9960859375 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9627291666666666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9614197916666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9620744791666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9805130208333332 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9776473958333334 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 779, - "fn": 21, - "accuracy": 0.97375 - } - }, - "auroc": 0.9790802083333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999145833333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99978125 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998479166666666 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999759375 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9967322916666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9982458333333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998369791666666 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9982567708333334 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999046875 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.99748125 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.99748125 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9920145833333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9920145833333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9947479166666666 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9947479166666666 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.989140625 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.989140625 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9882770833333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9882770833333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9887088541666667 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9887088541666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999187500000001 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999187500000001 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997833333333334 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997833333333334 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998510416666666 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998510416666666 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997729166666667 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997729166666667 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993489583333333 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993489583333333 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995609375 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995609375 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9962041666666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9962041666666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9963677083333333 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9963677083333333 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9962859374999999 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9962859374999999 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2195, - "fn": 5, - "accuracy": 0.9977272727272727 - } - }, - "auroc": 0.9970068181818181 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1194, - "fn": 6, - "accuracy": 0.995 - } - }, - "auroc": 0.9921600694444445 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3389, - "fn": 11, - "accuracy": 0.9967647058823529 - } - }, - "auroc": 0.9952962009803922 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2198, - "fn": 2, - "accuracy": 0.9990909090909091 - }, - "0.01": { - "tp": 2162, - "fn": 38, - "accuracy": 0.9827272727272728 - } - }, - "auroc": 0.9876007575757576 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1174, - "fn": 26, - "accuracy": 0.9783333333333334 - } - }, - "auroc": 0.9773256944444444 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3395, - "fn": 5, - "accuracy": 0.9985294117647059 - }, - "0.01": { - "tp": 3336, - "fn": 64, - "accuracy": 0.9811764705882353 - } - }, - "auroc": 0.9839742647058823 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 4398, - "fn": 2, - "accuracy": 0.9995454545454545 - }, - "0.01": { - "tp": 4357, - "fn": 43, - "accuracy": 0.9902272727272727 - } - }, - "auroc": 0.9923037878787879 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2397, - "fn": 3, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 2368, - "fn": 32, - "accuracy": 0.9866666666666667 - } - }, - "auroc": 0.9847428819444444 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 6795, - "fn": 5, - "accuracy": 0.9992647058823529 - }, - "0.01": { - "tp": 6725, - "fn": 75, - "accuracy": 0.9889705882352942 - } - }, - "auroc": 0.9896352328431373 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999343749999999 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997364583333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998354166666666 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996822916666667 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996916666666666 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996869791666666 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998083333333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997140625 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997611979166666 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9933968750000001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.991103125 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9922500000000001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9836354166666668 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9798989583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9817671875 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9885161458333334 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9855010416666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 789, - "fn": 11, - "accuracy": 0.98625 - } - }, - "auroc": 0.98700859375 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997541666666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9899520833333333 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.994853125 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997625 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.98038125 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.990071875 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997583333333334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9851666666666665 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9924625 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9953916666666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9917791666666668 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9935854166666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.966475 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9671072916666666 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9667911458333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9809333333333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9794432291666668 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 783, - "fn": 17, - "accuracy": 0.97875 - } - }, - "auroc": 0.98018828125 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983104166666666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9977604166666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980354166666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9691354166666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.963753125 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9664442708333334 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9837229166666668 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9807567708333333 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 777, - "fn": 23, - "accuracy": 0.97125 - } - }, - "auroc": 0.9822398437500001 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998864583333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999197916666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999903125 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998666666666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986552083333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992609375 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998765625 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992875 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99958203125 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9977302083333334 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9977302083333334 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9938677083333334 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9938677083333334 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9957989583333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9957989583333333 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9926197916666667 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9926197916666667 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.992309375 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.992309375 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9924645833333333 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9924645833333333 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999177083333334 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999177083333334 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999135416666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999135416666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999915625 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999915625 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998739583333334 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998739583333334 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998760416666667 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998760416666667 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999875 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999875 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980177083333334 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980177083333334 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9975729166666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9975729166666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9977953125 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9977953125 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2199, - "fn": 1, - "accuracy": 0.9995454545454545 - } - }, - "auroc": 0.9977121212121212 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - } - }, - "auroc": 0.9950418402777778 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3396, - "fn": 4, - "accuracy": 0.9988235294117647 - } - }, - "auroc": 0.9967696691176471 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2190, - "fn": 10, - "accuracy": 0.9954545454545455 - }, - "0.01": { - "tp": 2168, - "fn": 32, - "accuracy": 0.9854545454545455 - } - }, - "auroc": 0.9910997159090909 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1179, - "fn": 21, - "accuracy": 0.9825 - } - }, - "auroc": 0.98158125 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3387, - "fn": 13, - "accuracy": 0.9961764705882353 - }, - "0.01": { - "tp": 3347, - "fn": 53, - "accuracy": 0.9844117647058823 - } - }, - "auroc": 0.9877402573529411 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 4390, - "fn": 10, - "accuracy": 0.9977272727272727 - }, - "0.01": { - "tp": 4367, - "fn": 33, - "accuracy": 0.9925 - } - }, - "auroc": 0.994405918560606 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2397, - "fn": 3, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 2376, - "fn": 24, - "accuracy": 0.99 - } - }, - "auroc": 0.9883115451388889 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 6787, - "fn": 13, - "accuracy": 0.9980882352941176 - }, - "0.01": { - "tp": 6743, - "fn": 57, - "accuracy": 0.9916176470588235 - } - }, - "auroc": 0.992254963235294 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.996875 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9957739583333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9963244791666667 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.997896875 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99599375 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9969453125000001 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9973859375 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9958838541666667 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9966348958333334 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9572958333333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.9193989583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 331, - "fn": 69, - "accuracy": 0.8275 - } - }, - "auroc": 0.9383473958333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.976671875 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - } - }, - "auroc": 0.9010979166666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 347, - "fn": 53, - "accuracy": 0.8675 - } - }, - "auroc": 0.9388848958333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9669838541666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 309, - "fn": 91, - "accuracy": 0.7725 - } - }, - "auroc": 0.9102484375 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 678, - "fn": 122, - "accuracy": 0.8475 - } - }, - "auroc": 0.9386161458333333 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9920302083333332 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9496520833333333 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9708411458333335 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9933885416666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.932446875 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9629177083333333 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9927093750000001 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 348, - "fn": 52, - "accuracy": 0.87 - } - }, - "auroc": 0.9410494791666668 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 747, - "fn": 53, - "accuracy": 0.93375 - } - }, - "auroc": 0.9668794270833333 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.9089354166666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.96313125 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 339, - "fn": 61, - "accuracy": 0.8475 - } - }, - "auroc": 0.9360333333333333 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9721593749999999 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9297625 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9509609375 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 340, - "fn": 60, - "accuracy": 0.85 - } - }, - "auroc": 0.9405473958333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.946446875 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 709, - "fn": 91, - "accuracy": 0.88625 - } - }, - "auroc": 0.9434971354166666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9612791666666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9530708333333333 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.957175 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9644854166666668 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9229583333333333 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.943721875 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9628822916666666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9380145833333333 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 730, - "fn": 70, - "accuracy": 0.9125 - } - }, - "auroc": 0.9504484375000001 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.99149375 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9940614583333334 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9927776041666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9928927083333334 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9819260416666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9874093749999999 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9921932291666666 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.98799375 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - } - }, - "auroc": 0.9900934895833333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9743979166666666 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9743979166666666 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9703697916666667 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9703697916666667 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9723838541666666 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9723838541666666 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9782041666666667 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9782041666666667 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.972403125 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.972403125 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9753036458333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9753036458333334 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9956635416666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9956635416666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9949666666666668 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9949666666666668 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9953151041666668 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9953151041666668 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9889781249999999 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9889781249999999 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9917802083333334 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9917802083333334 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9903791666666666 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9903791666666666 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9894604166666666 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9894604166666666 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9819 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9819 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9856802083333334 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9856802083333334 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2199, - "fn": 1, - "accuracy": 0.9995454545454545 - }, - "0.01": { - "tp": 2092, - "fn": 108, - "accuracy": 0.9509090909090909 - } - }, - "auroc": 0.9758739583333333 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1199, - "fn": 1, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 1116, - "fn": 84, - "accuracy": 0.93 - } - }, - "auroc": 0.9625147569444444 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3398, - "fn": 2, - "accuracy": 0.9994117647058823 - }, - "0.01": { - "tp": 3208, - "fn": 192, - "accuracy": 0.9435294117647058 - } - }, - "auroc": 0.9711589460784313 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2198, - "fn": 2, - "accuracy": 0.9990909090909091 - }, - "0.01": { - "tp": 2168, - "fn": 32, - "accuracy": 0.9854545454545455 - } - }, - "auroc": 0.9826285984848485 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1196, - "fn": 4, - "accuracy": 0.9966666666666667 - }, - "0.01": { - "tp": 1056, - "fn": 144, - "accuracy": 0.88 - } - }, - "auroc": 0.9440309027777778 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3394, - "fn": 6, - "accuracy": 0.9982352941176471 - }, - "0.01": { - "tp": 3224, - "fn": 176, - "accuracy": 0.9482352941176471 - } - }, - "auroc": 0.9690058823529412 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 4397, - "fn": 3, - "accuracy": 0.9993181818181818 - }, - "0.01": { - "tp": 4260, - "fn": 140, - "accuracy": 0.9681818181818181 - } - }, - "auroc": 0.9792512784090909 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2395, - "fn": 5, - "accuracy": 0.9979166666666667 - }, - "0.01": { - "tp": 2172, - "fn": 228, - "accuracy": 0.905 - } - }, - "auroc": 0.9532728298611111 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 6792, - "fn": 8, - "accuracy": 0.9988235294117647 - }, - "0.01": { - "tp": 6432, - "fn": 368, - "accuracy": 0.9458823529411765 - } - }, - "auroc": 0.9700824142156862 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999260416666667 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99975625 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998411458333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996843750000001 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996677083333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996760416666667 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998052083333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997119791666667 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997585937500001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9867343749999999 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9907041666666666 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9887192708333332 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.97449375 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9795114583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9770026041666668 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9806140624999999 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9851078125 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 780, - "fn": 20, - "accuracy": 0.975 - } - }, - "auroc": 0.9828609375 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99971875 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9896822916666665 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9947005208333335 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997124999999999 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9800312499999999 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.989871875 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999715625 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9848567708333333 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9922861979166666 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9856833333333332 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.979125 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9824041666666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9502145833333333 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9556302083333335 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9529223958333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9679489583333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9673776041666666 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 766, - "fn": 34, - "accuracy": 0.9575 - } - }, - "auroc": 0.9676632812499999 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9960614583333334 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9969697916666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.996515625 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.96364375 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9568979166666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9602708333333333 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9798526041666666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9769338541666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 771, - "fn": 29, - "accuracy": 0.96375 - } - }, - "auroc": 0.9783932291666666 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99985 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99981875 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998343749999999 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998479166666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9984395833333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99914375 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998489583333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9991291666666666 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994890625 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.99780625 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.99780625 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.992328125 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.992328125 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9950671875 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9950671875 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9914989583333333 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9914989583333333 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9917479166666667 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9917479166666667 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9916234374999999 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9916234374999999 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998760416666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998760416666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998677083333333 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998677083333333 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999871875 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999871875 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99979375 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99979375 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997958333333333 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997958333333333 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997947916666666 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997947916666666 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9976125 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9976125 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9970604166666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9970604166666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9973364583333333 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9973364583333333 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2188, - "fn": 12, - "accuracy": 0.9945454545454545 - } - }, - "auroc": 0.9958692234848484 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1193, - "fn": 7, - "accuracy": 0.9941666666666666 - } - }, - "auroc": 0.9926760416666667 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3381, - "fn": 19, - "accuracy": 0.9944117647058823 - } - }, - "auroc": 0.9947422181372549 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2188, - "fn": 12, - "accuracy": 0.9945454545454545 - }, - "0.01": { - "tp": 2153, - "fn": 47, - "accuracy": 0.9786363636363636 - } - }, - "auroc": 0.9880360795454546 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1196, - "fn": 4, - "accuracy": 0.9966666666666667 - }, - "0.01": { - "tp": 1175, - "fn": 25, - "accuracy": 0.9791666666666666 - } - }, - "auroc": 0.9783630208333334 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3384, - "fn": 16, - "accuracy": 0.9952941176470588 - }, - "0.01": { - "tp": 3328, - "fn": 72, - "accuracy": 0.9788235294117648 - } - }, - "auroc": 0.9846220588235294 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 4388, - "fn": 12, - "accuracy": 0.9972727272727273 - }, - "0.01": { - "tp": 4341, - "fn": 59, - "accuracy": 0.9865909090909091 - } - }, - "auroc": 0.9919526515151516 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 2368, - "fn": 32, - "accuracy": 0.9866666666666667 - } - }, - "auroc": 0.98551953125 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 6784, - "fn": 16, - "accuracy": 0.9976470588235294 - }, - "0.01": { - "tp": 6709, - "fn": 91, - "accuracy": 0.9866176470588235 - } - }, - "auroc": 0.9896821384803921 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997552083333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997020833333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997286458333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999534375 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995385416666667 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995364583333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996447916666668 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996203124999999 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996325520833333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9932395833333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.990784375 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9920119791666666 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9711260416666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.980253125 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9756895833333334 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9821828125000001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.98551875 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 785, - "fn": 15, - "accuracy": 0.98125 - } - }, - "auroc": 0.98385078125 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996135416666666 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9901833333333333 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9948984375 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996010416666666 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9804510416666666 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9900260416666666 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996072916666666 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9853171875000001 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9924622395833333 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9952927083333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9837520833333333 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9895223958333333 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9463270833333333 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9523406249999999 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9493338541666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9708098958333333 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9680463541666666 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 760, - "fn": 40, - "accuracy": 0.95 - } - }, - "auroc": 0.969428125 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981322916666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9959520833333333 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9970421875000001 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9555479166666666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9534541666666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9545010416666666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9768401041666666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.974703125 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 789, - "fn": 11, - "accuracy": 0.98625 - }, - "0.01": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - } - }, - "auroc": 0.9757716145833334 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999725 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999721875 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997234375 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997083333333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9984447916666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990765625 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997166666666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990833333333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994000000000001 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9975302083333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9975302083333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9918479166666667 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9918479166666667 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9946890625 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9946890625 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9936208333333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9936208333333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9932458333333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9932458333333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9934333333333333 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9934333333333333 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997843750000001 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997843750000001 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997614583333333 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997614583333333 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997729166666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997729166666667 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996708333333333 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996708333333333 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996125 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996125 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996416666666667 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996416666666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9978760416666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9978760416666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9970822916666666 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9970822916666666 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9974791666666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9974791666666667 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2198, - "fn": 2, - "accuracy": 0.9990909090909091 - } - }, - "auroc": 0.9976582386363636 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1191, - "fn": 9, - "accuracy": 0.9925 - } - }, - "auroc": 0.9933493055555555 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3389, - "fn": 11, - "accuracy": 0.9967647058823529 - } - }, - "auroc": 0.9961374387254902 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2183, - "fn": 17, - "accuracy": 0.9922727272727273 - }, - "0.01": { - "tp": 2143, - "fn": 57, - "accuracy": 0.9740909090909091 - } - }, - "auroc": 0.9866722537878788 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1194, - "fn": 6, - "accuracy": 0.995 - }, - "0.01": { - "tp": 1172, - "fn": 28, - "accuracy": 0.9766666666666667 - } - }, - "auroc": 0.9774137152777778 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3377, - "fn": 23, - "accuracy": 0.9932352941176471 - }, - "0.01": { - "tp": 3315, - "fn": 85, - "accuracy": 0.975 - } - }, - "auroc": 0.9834045343137254 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 4383, - "fn": 17, - "accuracy": 0.9961363636363636 - }, - "0.01": { - "tp": 4341, - "fn": 59, - "accuracy": 0.9865909090909091 - } - }, - "auroc": 0.9921652462121212 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2394, - "fn": 6, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 2363, - "fn": 37, - "accuracy": 0.9845833333333334 - } - }, - "auroc": 0.9853815104166667 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 6777, - "fn": 23, - "accuracy": 0.9966176470588235 - }, - "0.01": { - "tp": 6704, - "fn": 96, - "accuracy": 0.9858823529411764 - } - }, - "auroc": 0.9897709865196078 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9640802083333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9550270833333332 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9595536458333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9658635416666665 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9543489583333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9601062499999999 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.964971875 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9546880208333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9598299479166666 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9955499999999999 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9804447916666665 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9879973958333332 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9858750000000001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9810604166666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9834677083333334 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9907125 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9807526041666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9857325520833333 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9872749999999999 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.965509375 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9763921875 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9862197916666666 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.977265625 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9817427083333334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9867473958333333 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9713875000000001 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9790674479166668 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9935708333333333 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9811104166666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.987340625 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.98333125 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9768802083333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9801057291666666 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9884510416666668 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9789953124999999 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9837231770833332 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9973427083333333 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9861010416666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9917218750000001 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9845406250000001 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9729208333333333 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9787307291666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9909416666666666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9795109375 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9852263020833334 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9784354166666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9661145833333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.972275 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9783927083333334 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9718333333333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9751130208333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9784140625 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9689739583333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9736940104166667 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9955208333333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9955208333333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9937229166666666 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9937229166666666 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9946218749999999 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9946218749999999 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9840666666666666 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9840666666666666 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9829645833333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9829645833333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.983515625 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.983515625 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9721114583333332 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9721114583333332 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9687749999999999 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9687749999999999 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9704432291666666 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9704432291666666 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.981425 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.981425 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9788322916666667 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9788322916666667 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9801286458333333 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9801286458333333 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9829072916666666 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9829072916666666 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9814385416666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9814385416666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9821729166666666 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9821729166666666 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9847532196969697 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1199, - "fn": 1, - "accuracy": 0.9991666666666666 - } - }, - "auroc": 0.9723845486111111 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3399, - "fn": 1, - "accuracy": 0.9997058823529412 - } - }, - "auroc": 0.980387806372549 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9809051136363637 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1195, - "fn": 5, - "accuracy": 0.9958333333333333 - } - }, - "auroc": 0.9723848958333334 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3395, - "fn": 5, - "accuracy": 0.9985294117647059 - } - }, - "auroc": 0.9778979779411765 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 4400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9828291666666666 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2394, - "fn": 6, - "accuracy": 0.9975 - } - }, - "auroc": 0.9723847222222223 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 6800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 6794, - "fn": 6, - "accuracy": 0.9991176470588236 - } - }, - "auroc": 0.9791428921568628 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998302083333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997229166666667 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997765624999999 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995864583333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99966875 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996276041666666 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997083333333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996958333333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997020833333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9924864583333334 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9907489583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9916177083333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.960984375 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9802375000000001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9706109375 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9767354166666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9854932291666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 789, - "fn": 11, - "accuracy": 0.98625 - } - }, - "auroc": 0.9811143229166668 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993291666666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9906927083333334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9950109375 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992083333333334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9805697916666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9898890625000001 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99926875 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.98563125 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9924499999999999 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.995221875 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.982125 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9886734374999999 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9378000000000001 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9557614583333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9467807291666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9665109375000001 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9689432291666668 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 765, - "fn": 35, - "accuracy": 0.95625 - } - }, - "auroc": 0.9677270833333333 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980135416666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9951708333333333 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9965921875 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.948784375 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9559552083333333 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9523697916666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9733989583333333 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9755630208333335 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 774, - "fn": 26, - "accuracy": 0.9675 - } - }, - "auroc": 0.9744809895833334 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996614583333334 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995895833333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996255208333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995624999999999 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998234375 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9988984375000001 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996119791666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989119791666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992619791666667 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9971197916666666 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9971197916666666 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9901958333333334 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9901958333333334 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9936578125000001 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9936578125000001 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9890635416666667 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9890635416666667 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9891468749999999 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9891468749999999 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9891052083333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9891052083333334 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995854166666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995854166666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995447916666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995447916666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995651041666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995651041666667 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993062500000001 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993062500000001 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987979166666666 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987979166666666 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990520833333334 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990520833333334 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9969072916666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9969072916666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9958145833333334 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9958145833333334 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9963609375 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9963609375 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2198, - "fn": 2, - "accuracy": 0.9990909090909091 - } - }, - "auroc": 0.9969568181818182 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1190, - "fn": 10, - "accuracy": 0.9916666666666667 - } - }, - "auroc": 0.9930083333333333 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3388, - "fn": 12, - "accuracy": 0.9964705882352941 - } - }, - "auroc": 0.9955632352941177 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2181, - "fn": 19, - "accuracy": 0.9913636363636363 - }, - "0.01": { - "tp": 2159, - "fn": 41, - "accuracy": 0.9813636363636363 - } - }, - "auroc": 0.9835841856060606 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1196, - "fn": 4, - "accuracy": 0.9966666666666667 - }, - "0.01": { - "tp": 1172, - "fn": 28, - "accuracy": 0.9766666666666667 - } - }, - "auroc": 0.9784045138888889 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3377, - "fn": 23, - "accuracy": 0.9932352941176471 - }, - "0.01": { - "tp": 3331, - "fn": 69, - "accuracy": 0.9797058823529412 - } - }, - "auroc": 0.9817560661764706 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 4381, - "fn": 19, - "accuracy": 0.9956818181818182 - }, - "0.01": { - "tp": 4357, - "fn": 43, - "accuracy": 0.9902272727272727 - } - }, - "auroc": 0.9902705018939394 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 2362, - "fn": 38, - "accuracy": 0.9841666666666666 - } - }, - "auroc": 0.985706423611111 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 6777, - "fn": 23, - "accuracy": 0.9966176470588235 - }, - "0.01": { - "tp": 6719, - "fn": 81, - "accuracy": 0.9880882352941176 - } - }, - "auroc": 0.9886596507352942 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999859375 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999703125 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99978125 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996062499999999 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997135416666667 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996598958333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997328125 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997083333333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997205729166667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9931802083333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9905937499999999 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9918869791666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9740697916666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9800875 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9770786458333334 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9836250000000001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.985340625 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 785, - "fn": 15, - "accuracy": 0.98125 - } - }, - "auroc": 0.9844828125 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995635416666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9904437500000001 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9950036458333333 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995208333333333 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9803895833333334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9899552083333334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995421875 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9854166666666666 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9924794270833334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.995284375 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.984090625 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9896875 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9493677083333333 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.955425 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9523963541666666 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9723260416666666 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9697578124999999 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 765, - "fn": 35, - "accuracy": 0.95625 - } - }, - "auroc": 0.9710419270833334 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981156250000001 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9959354166666666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9970255208333333 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9586718750000001 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9563145833333335 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9574932291666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9783937500000001 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.976125 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 771, - "fn": 29, - "accuracy": 0.96375 - } - }, - "auroc": 0.977259375 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997375 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997020833333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997197916666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996802083333334 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981458333333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989130208333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997088541666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989239583333334 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99931640625 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.997428125 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.997428125 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9916666666666667 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9916666666666667 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9945473958333334 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9945473958333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9910593750000001 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9910593750000001 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9912520833333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9912520833333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9911557291666666 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9911557291666666 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997541666666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997541666666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997291666666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997291666666667 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997416666666666 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997416666666666 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996260416666667 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996260416666667 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996145833333333 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996145833333333 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996203124999999 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996203124999999 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9975125 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9975125 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.99669375 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.99669375 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9971031250000001 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9971031250000001 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2198, - "fn": 2, - "accuracy": 0.9990909090909091 - } - }, - "auroc": 0.9973746212121211 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1192, - "fn": 8, - "accuracy": 0.9933333333333333 - } - }, - "auroc": 0.9934114583333333 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3390, - "fn": 10, - "accuracy": 0.9970588235294118 - } - }, - "auroc": 0.9959758578431374 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2183, - "fn": 17, - "accuracy": 0.9922727272727273 - }, - "0.01": { - "tp": 2148, - "fn": 52, - "accuracy": 0.9763636363636363 - } - }, - "auroc": 0.9872611742424242 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1195, - "fn": 5, - "accuracy": 0.9958333333333333 - }, - "0.01": { - "tp": 1174, - "fn": 26, - "accuracy": 0.9783333333333334 - } - }, - "auroc": 0.9783460069444444 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3378, - "fn": 22, - "accuracy": 0.9935294117647059 - }, - "0.01": { - "tp": 3322, - "fn": 78, - "accuracy": 0.9770588235294118 - } - }, - "auroc": 0.984114644607843 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 4383, - "fn": 17, - "accuracy": 0.9961363636363636 - }, - "0.01": { - "tp": 4346, - "fn": 54, - "accuracy": 0.9877272727272727 - } - }, - "auroc": 0.9923178977272726 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2395, - "fn": 5, - "accuracy": 0.9979166666666667 - }, - "0.01": { - "tp": 2366, - "fn": 34, - "accuracy": 0.9858333333333333 - } - }, - "auroc": 0.9858787326388889 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 6778, - "fn": 22, - "accuracy": 0.9967647058823529 - }, - "0.01": { - "tp": 6712, - "fn": 88, - "accuracy": 0.9870588235294118 - } - }, - "auroc": 0.9900452512254903 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998552083333334 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996958333333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997755208333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995947916666666 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996473958333333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997250000000001 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996979166666666 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997114583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9931989583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9907489583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9919739583333333 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.971353125 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9802375000000001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9757953125000001 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9822760416666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9854932291666667 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 785, - "fn": 15, - "accuracy": 0.98125 - } - }, - "auroc": 0.9838846354166667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995531249999999 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9906635416666666 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9951083333333334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995177083333334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9804197916666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9899687500000001 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995354166666667 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9855416666666668 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9925385416666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.995271875 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9837218750000001 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9894968749999999 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9466041666666667 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9528583333333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.94973125 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9709380208333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9682901041666666 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 760, - "fn": 40, - "accuracy": 0.95 - } - }, - "auroc": 0.9696140625 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981187499999999 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99576875 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99694375 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9558375000000001 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9547291666666666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9552833333333334 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.976978125 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9752489583333334 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - } - }, - "auroc": 0.9761135416666669 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997322916666667 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996989583333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999715625 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997083333333333 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981416666666666 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998925 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997203125 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989203125 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993203125 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974645833333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974645833333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9917427083333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9917427083333333 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9946036458333334 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9946036458333334 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9910645833333335 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9910645833333335 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9911687499999999 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9911687499999999 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9911166666666666 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9911166666666666 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999740625 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999740625 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997125 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997125 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997265625 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997265625 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999615625 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999615625 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995770833333334 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995770833333334 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995963541666666 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995963541666666 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974927083333334 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9974927083333334 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.996684375 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.996684375 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9970885416666667 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9970885416666667 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2198, - "fn": 2, - "accuracy": 0.9990909090909091 - } - }, - "auroc": 0.9973734848484848 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1191, - "fn": 9, - "accuracy": 0.9925 - } - }, - "auroc": 0.9933829861111111 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3389, - "fn": 11, - "accuracy": 0.9967647058823529 - } - }, - "auroc": 0.9959650735294117 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2183, - "fn": 17, - "accuracy": 0.9922727272727273 - }, - "0.01": { - "tp": 2143, - "fn": 57, - "accuracy": 0.9740909090909091 - } - }, - "auroc": 0.9865000946969698 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1195, - "fn": 5, - "accuracy": 0.9958333333333333 - }, - "0.01": { - "tp": 1172, - "fn": 28, - "accuracy": 0.9766666666666667 - } - }, - "auroc": 0.9776810763888888 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3378, - "fn": 22, - "accuracy": 0.9935294117647059 - }, - "0.01": { - "tp": 3315, - "fn": 85, - "accuracy": 0.975 - } - }, - "auroc": 0.9833874999999999 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 4383, - "fn": 17, - "accuracy": 0.9961363636363636 - }, - "0.01": { - "tp": 4341, - "fn": 59, - "accuracy": 0.9865909090909091 - } - }, - "auroc": 0.9919367897727271 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2395, - "fn": 5, - "accuracy": 0.9979166666666667 - }, - "0.01": { - "tp": 2363, - "fn": 37, - "accuracy": 0.9845833333333334 - } - }, - "auroc": 0.98553203125 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 6778, - "fn": 22, - "accuracy": 0.9967647058823529 - }, - "0.01": { - "tp": 6704, - "fn": 96, - "accuracy": 0.9858823529411764 - } - }, - "auroc": 0.9896762867647059 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9966466145833333 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9956627604166666 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9961546874999999 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9966699652777777 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9955697048611111 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9961198350694445 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9966582899305556 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9956162326388889 - }, - { - "domain": "recipes", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 9600, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9961372612847221 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2364, - "fn": 36, - "accuracy": 0.985 - } - }, - "auroc": 0.9899311631944445 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2328, - "fn": 72, - "accuracy": 0.97 - } - }, - "auroc": 0.98365234375 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - }, - "0.01": { - "tp": 4692, - "fn": 108, - "accuracy": 0.9775 - } - }, - "auroc": 0.9867917534722221 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2376, - "fn": 24, - "accuracy": 0.99 - }, - "0.01": { - "tp": 2342, - "fn": 58, - "accuracy": 0.9758333333333333 - } - }, - "auroc": 0.9758474826388889 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2297, - "fn": 103, - "accuracy": 0.9570833333333333 - } - }, - "auroc": 0.9731499131944443 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4775, - "fn": 25, - "accuracy": 0.9947916666666666 - }, - "0.01": { - "tp": 4639, - "fn": 161, - "accuracy": 0.9664583333333333 - } - }, - "auroc": 0.9744986979166668 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4776, - "fn": 24, - "accuracy": 0.995 - }, - "0.01": { - "tp": 4706, - "fn": 94, - "accuracy": 0.9804166666666667 - } - }, - "auroc": 0.9828893229166666 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4798, - "fn": 2, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 4625, - "fn": 175, - "accuracy": 0.9635416666666666 - } - }, - "auroc": 0.9784011284722222 - }, - { - "domain": "recipes", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9574, - "fn": 26, - "accuracy": 0.9972916666666667 - }, - "0.01": { - "tp": 9331, - "fn": 269, - "accuracy": 0.9719791666666666 - } - }, - "auroc": 0.9806452256944443 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9979736111111112 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2379, - "fn": 21, - "accuracy": 0.99125 - } - }, - "auroc": 0.9844664930555556 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4778, - "fn": 22, - "accuracy": 0.9954166666666666 - } - }, - "auroc": 0.9912200520833334 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9979725694444446 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2369, - "fn": 31, - "accuracy": 0.9870833333333333 - } - }, - "auroc": 0.9754464409722223 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - }, - "0.01": { - "tp": 4769, - "fn": 31, - "accuracy": 0.9935416666666667 - } - }, - "auroc": 0.9867095052083333 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - } - }, - "auroc": 0.9979730902777778 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - }, - "0.01": { - "tp": 4748, - "fn": 52, - "accuracy": 0.9891666666666666 - } - }, - "auroc": 0.9799564670138888 - }, - { - "domain": "recipes", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9599, - "fn": 1, - "accuracy": 0.9998958333333333 - }, - "0.01": { - "tp": 9547, - "fn": 53, - "accuracy": 0.9944791666666667 - } - }, - "auroc": 0.9889647786458333 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2343, - "fn": 57, - "accuracy": 0.97625 - } - }, - "auroc": 0.9872155381944445 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2347, - "fn": 53, - "accuracy": 0.9779166666666667 - } - }, - "auroc": 0.98265 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - }, - "0.01": { - "tp": 4690, - "fn": 110, - "accuracy": 0.9770833333333333 - } - }, - "auroc": 0.9849327690972223 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2352, - "fn": 48, - "accuracy": 0.98 - }, - "0.01": { - "tp": 2215, - "fn": 185, - "accuracy": 0.9229166666666667 - } - }, - "auroc": 0.9563761284722221 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2385, - "fn": 15, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 2280, - "fn": 120, - "accuracy": 0.95 - } - }, - "auroc": 0.956909375 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4737, - "fn": 63, - "accuracy": 0.986875 - }, - "0.01": { - "tp": 4495, - "fn": 305, - "accuracy": 0.9364583333333333 - } - }, - "auroc": 0.956642751736111 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4751, - "fn": 49, - "accuracy": 0.9897916666666666 - }, - "0.01": { - "tp": 4558, - "fn": 242, - "accuracy": 0.9495833333333333 - } - }, - "auroc": 0.9717958333333334 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4785, - "fn": 15, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 4627, - "fn": 173, - "accuracy": 0.9639583333333334 - } - }, - "auroc": 0.9697796874999999 - }, - { - "domain": "recipes", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9536, - "fn": 64, - "accuracy": 0.9933333333333333 - }, - "0.01": { - "tp": 9185, - "fn": 415, - "accuracy": 0.9567708333333333 - } - }, - "auroc": 0.9707877604166666 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2385, - "fn": 15, - "accuracy": 0.99375 - } - }, - "auroc": 0.9949112847222222 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2387, - "fn": 13, - "accuracy": 0.9945833333333334 - } - }, - "auroc": 0.991778125 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4772, - "fn": 28, - "accuracy": 0.9941666666666666 - } - }, - "auroc": 0.993344704861111 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2349, - "fn": 51, - "accuracy": 0.97875 - }, - "0.01": { - "tp": 2237, - "fn": 163, - "accuracy": 0.9320833333333334 - } - }, - "auroc": 0.9629907118055556 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2373, - "fn": 27, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 2254, - "fn": 146, - "accuracy": 0.9391666666666667 - } - }, - "auroc": 0.9563926215277778 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4722, - "fn": 78, - "accuracy": 0.98375 - }, - "0.01": { - "tp": 4491, - "fn": 309, - "accuracy": 0.935625 - } - }, - "auroc": 0.9596916666666667 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4749, - "fn": 51, - "accuracy": 0.989375 - }, - "0.01": { - "tp": 4622, - "fn": 178, - "accuracy": 0.9629166666666666 - } - }, - "auroc": 0.9789509982638889 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4773, - "fn": 27, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 4641, - "fn": 159, - "accuracy": 0.966875 - } - }, - "auroc": 0.9740853732638889 - }, - { - "domain": "recipes", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9522, - "fn": 78, - "accuracy": 0.991875 - }, - "0.01": { - "tp": 9263, - "fn": 337, - "accuracy": 0.9648958333333333 - } - }, - "auroc": 0.9765181857638889 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2397, - "fn": 3, - "accuracy": 0.99875 - } - }, - "auroc": 0.997322829861111 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9964829861111111 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4797, - "fn": 3, - "accuracy": 0.999375 - } - }, - "auroc": 0.9969029079861111 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9974000000000001 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2394, - "fn": 6, - "accuracy": 0.9975 - } - }, - "auroc": 0.9946500868055556 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4794, - "fn": 6, - "accuracy": 0.99875 - } - }, - "auroc": 0.9960250434027778 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4797, - "fn": 3, - "accuracy": 0.999375 - } - }, - "auroc": 0.9973614149305555 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4794, - "fn": 6, - "accuracy": 0.99875 - } - }, - "auroc": 0.9955665364583334 - }, - { - "domain": "recipes", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 9591, - "fn": 9, - "accuracy": 0.9990625 - } - }, - "auroc": 0.9964639756944444 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2385, - "fn": 15, - "accuracy": 0.99375 - } - }, - "auroc": 0.9954869791666667 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2385, - "fn": 15, - "accuracy": 0.99375 - } - }, - "auroc": 0.9954869791666667 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2356, - "fn": 44, - "accuracy": 0.9816666666666667 - } - }, - "auroc": 0.9905184895833334 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2356, - "fn": 44, - "accuracy": 0.9816666666666667 - } - }, - "auroc": 0.9905184895833334 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4741, - "fn": 59, - "accuracy": 0.9877083333333333 - } - }, - "auroc": 0.9930027343749999 - }, - { - "domain": "recipes", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4741, - "fn": 59, - "accuracy": 0.9877083333333333 - } - }, - "auroc": 0.9930027343749999 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9896457465277777 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9896457465277777 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2387, - "fn": 13, - "accuracy": 0.9945833333333334 - } - }, - "auroc": 0.9890875000000001 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2387, - "fn": 13, - "accuracy": 0.9945833333333334 - } - }, - "auroc": 0.9890875000000001 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4786, - "fn": 14, - "accuracy": 0.9970833333333333 - } - }, - "auroc": 0.9893666232638888 - }, - { - "domain": "recipes", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4786, - "fn": 14, - "accuracy": 0.9970833333333333 - } - }, - "auroc": 0.9893666232638888 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.997151388888889 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.997151388888889 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9967855034722222 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9967855034722222 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - } - }, - "auroc": 0.9969684461805555 - }, - { - "domain": "recipes", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - } - }, - "auroc": 0.9969684461805555 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9972739583333332 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9972739583333332 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - } - }, - "auroc": 0.9971981770833332 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - } - }, - "auroc": 0.9971981770833332 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4797, - "fn": 3, - "accuracy": 0.999375 - } - }, - "auroc": 0.9972360677083334 - }, - { - "domain": "recipes", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4797, - "fn": 3, - "accuracy": 0.999375 - } - }, - "auroc": 0.9972360677083334 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2389, - "fn": 11, - "accuracy": 0.9954166666666666 - } - }, - "auroc": 0.9956133680555554 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2389, - "fn": 11, - "accuracy": 0.9954166666666666 - } - }, - "auroc": 0.9956133680555554 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2393, - "fn": 7, - "accuracy": 0.9970833333333333 - }, - "0.01": { - "tp": 2388, - "fn": 12, - "accuracy": 0.995 - } - }, - "auroc": 0.9943738715277778 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2393, - "fn": 7, - "accuracy": 0.9970833333333333 - }, - "0.01": { - "tp": 2388, - "fn": 12, - "accuracy": 0.995 - } - }, - "auroc": 0.9943738715277778 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4793, - "fn": 7, - "accuracy": 0.9985416666666667 - }, - "0.01": { - "tp": 4777, - "fn": 23, - "accuracy": 0.9952083333333334 - } - }, - "auroc": 0.9949936197916666 - }, - { - "domain": "recipes", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4793, - "fn": 7, - "accuracy": 0.9985416666666667 - }, - "0.01": { - "tp": 4777, - "fn": 23, - "accuracy": 0.9952083333333334 - } - }, - "auroc": 0.9949936197916666 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 26399, - "fn": 1, - "accuracy": 0.9999621212121212 - }, - "0.01": { - "tp": 26260, - "fn": 140, - "accuracy": 0.9946969696969697 - } - }, - "auroc": 0.9944702256944444 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 14399, - "fn": 1, - "accuracy": 0.9999305555555555 - }, - "0.01": { - "tp": 14241, - "fn": 159, - "accuracy": 0.9889583333333334 - } - }, - "auroc": 0.989115451388889 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 40798, - "fn": 2, - "accuracy": 0.9999509803921569 - }, - "0.01": { - "tp": 40501, - "fn": 299, - "accuracy": 0.992671568627451 - } - }, - "auroc": 0.9925803053513073 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 26270, - "fn": 130, - "accuracy": 0.9950757575757576 - }, - "0.01": { - "tp": 25922, - "fn": 478, - "accuracy": 0.9818939393939394 - } - }, - "auroc": 0.9868382181186869 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 14356, - "fn": 44, - "accuracy": 0.9969444444444444 - }, - "0.01": { - "tp": 13994, - "fn": 406, - "accuracy": 0.9718055555555556 - } - }, - "auroc": 0.9753530237268518 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 40626, - "fn": 174, - "accuracy": 0.995735294117647 - }, - "0.01": { - "tp": 39916, - "fn": 884, - "accuracy": 0.9783333333333334 - } - }, - "auroc": 0.9827846200980391 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 52669, - "fn": 131, - "accuracy": 0.9975189393939394 - }, - "0.01": { - "tp": 52182, - "fn": 618, - "accuracy": 0.9882954545454545 - } - }, - "auroc": 0.9906542219065657 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 28755, - "fn": 45, - "accuracy": 0.9984375 - }, - "0.01": { - "tp": 28235, - "fn": 565, - "accuracy": 0.9803819444444445 - } - }, - "auroc": 0.9822342375578704 - }, - { - "domain": "recipes", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 81424, - "fn": 176, - "accuracy": 0.9978431372549019 - }, - "0.01": { - "tp": 80417, - "fn": 1183, - "accuracy": 0.9855024509803921 - } - }, - "auroc": 0.9876824627246732 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999940625 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994770833333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997088541666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998208333333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.99056875 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9951947916666668 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998807291666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9950229166666666 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - } - }, - "auroc": 0.9974518229166667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9655302083333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9894302083333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9774802083333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 146, - "fn": 54, - "accuracy": 0.73 - } - }, - "auroc": 0.9204427083333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9737885416666667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 336, - "fn": 64, - "accuracy": 0.84 - } - }, - "auroc": 0.947115625 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9429864583333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.981609375 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 717, - "fn": 83, - "accuracy": 0.89625 - } - }, - "auroc": 0.9622979166666668 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9792197916666667 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9814343750000001 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9803270833333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9701333333333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9613333333333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.9657333333333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9746765625 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9713838541666666 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 736, - "fn": 64, - "accuracy": 0.92 - } - }, - "auroc": 0.9730302083333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9402791666666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981552083333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9692171875 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 124, - "fn": 76, - "accuracy": 0.62 - } - }, - "auroc": 0.9045125 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9606302083333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 310, - "fn": 90, - "accuracy": 0.775 - } - }, - "auroc": 0.9325713541666666 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 301, - "fn": 99, - "accuracy": 0.7525 - } - }, - "auroc": 0.9223958333333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9793927083333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 771, - "fn": 29, - "accuracy": 0.96375 - }, - "0.01": { - "tp": 687, - "fn": 113, - "accuracy": 0.85875 - } - }, - "auroc": 0.9508942708333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.989121875 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9921854166666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9906536458333335 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 154, - "fn": 46, - "accuracy": 0.77 - }, - "0.01": { - "tp": 109, - "fn": 91, - "accuracy": 0.545 - } - }, - "auroc": 0.8693427083333332 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9712729166666666 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 348, - "fn": 52, - "accuracy": 0.87 - }, - "0.01": { - "tp": 297, - "fn": 103, - "accuracy": 0.7425 - } - }, - "auroc": 0.9203078124999999 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - }, - "0.01": { - "tp": 306, - "fn": 94, - "accuracy": 0.765 - } - }, - "auroc": 0.9292322916666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9817291666666668 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 747, - "fn": 53, - "accuracy": 0.93375 - }, - "0.01": { - "tp": 692, - "fn": 108, - "accuracy": 0.865 - } - }, - "auroc": 0.9554807291666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.99126875 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9937770833333335 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9925229166666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.987225 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9757291666666666 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9814770833333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9892468750000001 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.984753125 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 786, - "fn": 14, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 774, - "fn": 26, - "accuracy": 0.9675 - } - }, - "auroc": 0.987 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9432739583333333 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9432739583333333 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9426343749999999 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9426343749999999 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9429541666666668 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9429541666666668 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 109, - "fn": 91, - "accuracy": 0.545 - } - }, - "auroc": 0.8594041666666667 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 109, - "fn": 91, - "accuracy": 0.545 - } - }, - "auroc": 0.8594041666666667 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - }, - "0.01": { - "tp": 98, - "fn": 102, - "accuracy": 0.49 - } - }, - "auroc": 0.8146375 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - }, - "0.01": { - "tp": 98, - "fn": 102, - "accuracy": 0.49 - } - }, - "auroc": 0.8146375 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 285, - "fn": 115, - "accuracy": 0.7125 - }, - "0.01": { - "tp": 207, - "fn": 193, - "accuracy": 0.5175 - } - }, - "auroc": 0.8370208333333334 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 285, - "fn": 115, - "accuracy": 0.7125 - }, - "0.01": { - "tp": 207, - "fn": 193, - "accuracy": 0.5175 - } - }, - "auroc": 0.8370208333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9977083333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9977083333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9921552083333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9921552083333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9949317708333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9949317708333334 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9927166666666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9927166666666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9543520833333333 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9543520833333333 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9735343750000001 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9735343750000001 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 125, - "fn": 75, - "accuracy": 0.625 - } - }, - "auroc": 0.8723614583333332 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 125, - "fn": 75, - "accuracy": 0.625 - } - }, - "auroc": 0.8723614583333332 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - }, - "0.01": { - "tp": 104, - "fn": 96, - "accuracy": 0.52 - } - }, - "auroc": 0.8267635416666665 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - }, - "0.01": { - "tp": 104, - "fn": 96, - "accuracy": 0.52 - } - }, - "auroc": 0.8267635416666665 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 281, - "fn": 119, - "accuracy": 0.7025 - }, - "0.01": { - "tp": 229, - "fn": 171, - "accuracy": 0.5725 - } - }, - "auroc": 0.8495625 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 281, - "fn": 119, - "accuracy": 0.7025 - }, - "0.01": { - "tp": 229, - "fn": 171, - "accuracy": 0.5725 - } - }, - "auroc": 0.8495625 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2063, - "fn": 137, - "accuracy": 0.9377272727272727 - }, - "0.01": { - "tp": 1940, - "fn": 260, - "accuracy": 0.8818181818181818 - } - }, - "auroc": 0.9573477272727273 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1189, - "fn": 11, - "accuracy": 0.9908333333333333 - }, - "0.01": { - "tp": 1179, - "fn": 21, - "accuracy": 0.9825 - } - }, - "auroc": 0.9924098958333334 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3252, - "fn": 148, - "accuracy": 0.9564705882352941 - }, - "0.01": { - "tp": 3119, - "fn": 281, - "accuracy": 0.9173529411764706 - } - }, - "auroc": 0.9697226102941177 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1932, - "fn": 268, - "accuracy": 0.8781818181818182 - }, - "0.01": { - "tp": 1688, - "fn": 512, - "accuracy": 0.7672727272727272 - } - }, - "auroc": 0.9256381628787878 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1167, - "fn": 33, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 1124, - "fn": 76, - "accuracy": 0.9366666666666666 - } - }, - "auroc": 0.9722204861111112 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3099, - "fn": 301, - "accuracy": 0.9114705882352941 - }, - "0.01": { - "tp": 2812, - "fn": 588, - "accuracy": 0.8270588235294117 - } - }, - "auroc": 0.9420789828431373 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3995, - "fn": 405, - "accuracy": 0.9079545454545455 - }, - "0.01": { - "tp": 3628, - "fn": 772, - "accuracy": 0.8245454545454546 - } - }, - "auroc": 0.9414929450757575 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2356, - "fn": 44, - "accuracy": 0.9816666666666667 - }, - "0.01": { - "tp": 2303, - "fn": 97, - "accuracy": 0.9595833333333333 - } - }, - "auroc": 0.9823151909722222 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 6351, - "fn": 449, - "accuracy": 0.9339705882352941 - }, - "0.01": { - "tp": 5931, - "fn": 869, - "accuracy": 0.8722058823529412 - } - }, - "auroc": 0.9559007965686273 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999940625 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994770833333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997088541666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998208333333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.99056875 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9951947916666668 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998807291666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9950229166666666 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - } - }, - "auroc": 0.9974518229166667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9655302083333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9894302083333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9774802083333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 146, - "fn": 54, - "accuracy": 0.73 - } - }, - "auroc": 0.9204427083333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9737885416666667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 336, - "fn": 64, - "accuracy": 0.84 - } - }, - "auroc": 0.947115625 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9429864583333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.981609375 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 717, - "fn": 83, - "accuracy": 0.89625 - } - }, - "auroc": 0.9622979166666668 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9792197916666667 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9814343750000001 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9803270833333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9701333333333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9613333333333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.9657333333333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9746765625 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9713838541666666 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 736, - "fn": 64, - "accuracy": 0.92 - } - }, - "auroc": 0.9730302083333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9402791666666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981552083333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9692171875 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 124, - "fn": 76, - "accuracy": 0.62 - } - }, - "auroc": 0.9045125 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9606302083333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 310, - "fn": 90, - "accuracy": 0.775 - } - }, - "auroc": 0.9325713541666666 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 301, - "fn": 99, - "accuracy": 0.7525 - } - }, - "auroc": 0.9223958333333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9793927083333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 771, - "fn": 29, - "accuracy": 0.96375 - }, - "0.01": { - "tp": 687, - "fn": 113, - "accuracy": 0.85875 - } - }, - "auroc": 0.9508942708333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.989121875 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9921854166666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9906536458333335 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 154, - "fn": 46, - "accuracy": 0.77 - }, - "0.01": { - "tp": 109, - "fn": 91, - "accuracy": 0.545 - } - }, - "auroc": 0.8693427083333332 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9712729166666666 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 348, - "fn": 52, - "accuracy": 0.87 - }, - "0.01": { - "tp": 297, - "fn": 103, - "accuracy": 0.7425 - } - }, - "auroc": 0.9203078124999999 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - }, - "0.01": { - "tp": 306, - "fn": 94, - "accuracy": 0.765 - } - }, - "auroc": 0.9292322916666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9817291666666668 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 747, - "fn": 53, - "accuracy": 0.93375 - }, - "0.01": { - "tp": 692, - "fn": 108, - "accuracy": 0.865 - } - }, - "auroc": 0.9554807291666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.99126875 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9937770833333335 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9925229166666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.987225 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9757291666666666 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9814770833333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9892468750000001 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.984753125 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 786, - "fn": 14, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 774, - "fn": 26, - "accuracy": 0.9675 - } - }, - "auroc": 0.987 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9432739583333333 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9432739583333333 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9426343749999999 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9426343749999999 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9429541666666668 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9429541666666668 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 109, - "fn": 91, - "accuracy": 0.545 - } - }, - "auroc": 0.8594041666666667 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 109, - "fn": 91, - "accuracy": 0.545 - } - }, - "auroc": 0.8594041666666667 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - }, - "0.01": { - "tp": 98, - "fn": 102, - "accuracy": 0.49 - } - }, - "auroc": 0.8146375 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - }, - "0.01": { - "tp": 98, - "fn": 102, - "accuracy": 0.49 - } - }, - "auroc": 0.8146375 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 285, - "fn": 115, - "accuracy": 0.7125 - }, - "0.01": { - "tp": 207, - "fn": 193, - "accuracy": 0.5175 - } - }, - "auroc": 0.8370208333333334 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 285, - "fn": 115, - "accuracy": 0.7125 - }, - "0.01": { - "tp": 207, - "fn": 193, - "accuracy": 0.5175 - } - }, - "auroc": 0.8370208333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9977083333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9977083333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9921552083333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9921552083333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9949317708333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9949317708333334 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9927166666666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9927166666666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9543520833333333 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9543520833333333 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9735343750000001 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9735343750000001 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 125, - "fn": 75, - "accuracy": 0.625 - } - }, - "auroc": 0.8723614583333332 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 125, - "fn": 75, - "accuracy": 0.625 - } - }, - "auroc": 0.8723614583333332 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - }, - "0.01": { - "tp": 104, - "fn": 96, - "accuracy": 0.52 - } - }, - "auroc": 0.8267635416666665 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - }, - "0.01": { - "tp": 104, - "fn": 96, - "accuracy": 0.52 - } - }, - "auroc": 0.8267635416666665 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 281, - "fn": 119, - "accuracy": 0.7025 - }, - "0.01": { - "tp": 229, - "fn": 171, - "accuracy": 0.5725 - } - }, - "auroc": 0.8495625 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 281, - "fn": 119, - "accuracy": 0.7025 - }, - "0.01": { - "tp": 229, - "fn": 171, - "accuracy": 0.5725 - } - }, - "auroc": 0.8495625 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2063, - "fn": 137, - "accuracy": 0.9377272727272727 - }, - "0.01": { - "tp": 1940, - "fn": 260, - "accuracy": 0.8818181818181818 - } - }, - "auroc": 0.9573477272727273 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1189, - "fn": 11, - "accuracy": 0.9908333333333333 - }, - "0.01": { - "tp": 1179, - "fn": 21, - "accuracy": 0.9825 - } - }, - "auroc": 0.9924098958333334 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3252, - "fn": 148, - "accuracy": 0.9564705882352941 - }, - "0.01": { - "tp": 3119, - "fn": 281, - "accuracy": 0.9173529411764706 - } - }, - "auroc": 0.9697226102941177 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1932, - "fn": 268, - "accuracy": 0.8781818181818182 - }, - "0.01": { - "tp": 1688, - "fn": 512, - "accuracy": 0.7672727272727272 - } - }, - "auroc": 0.9256381628787878 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1167, - "fn": 33, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 1124, - "fn": 76, - "accuracy": 0.9366666666666666 - } - }, - "auroc": 0.9722204861111112 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3099, - "fn": 301, - "accuracy": 0.9114705882352941 - }, - "0.01": { - "tp": 2812, - "fn": 588, - "accuracy": 0.8270588235294117 - } - }, - "auroc": 0.9420789828431373 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3995, - "fn": 405, - "accuracy": 0.9079545454545455 - }, - "0.01": { - "tp": 3628, - "fn": 772, - "accuracy": 0.8245454545454546 - } - }, - "auroc": 0.9414929450757575 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2356, - "fn": 44, - "accuracy": 0.9816666666666667 - }, - "0.01": { - "tp": 2303, - "fn": 97, - "accuracy": 0.9595833333333333 - } - }, - "auroc": 0.9823151909722222 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 6351, - "fn": 449, - "accuracy": 0.9339705882352941 - }, - "0.01": { - "tp": 5931, - "fn": 869, - "accuracy": 0.8722058823529412 - } - }, - "auroc": 0.9559007965686273 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998697916666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999440625 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996552083333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996739583333334 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9964781250000001 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980760416666666 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999771875 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9979593750000001 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9988656250000001 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9917239583333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9906427083333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9911833333333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9729208333333332 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9711479166666667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.972034375 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9823223958333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9808953125 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 784, - "fn": 16, - "accuracy": 0.98 - }, - "0.01": { - "tp": 775, - "fn": 25, - "accuracy": 0.96875 - } - }, - "auroc": 0.9816088541666667 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9838708333333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9854229166666668 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9846468749999999 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9746552083333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.968978125 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - } - }, - "auroc": 0.9718166666666668 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9792630208333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9772005208333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 772, - "fn": 28, - "accuracy": 0.965 - }, - "0.01": { - "tp": 757, - "fn": 43, - "accuracy": 0.94625 - } - }, - "auroc": 0.9782317708333332 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.98665625 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989260416666668 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9927911458333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9700010416666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.98683125 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9784161458333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9783286458333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9928786458333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 784, - "fn": 16, - "accuracy": 0.98 - } - }, - "auroc": 0.9856036458333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9975854166666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9953916666666668 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9964885416666668 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9487239583333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9799572916666666 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9643406250000001 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9731546875000001 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9876744791666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 788, - "fn": 12, - "accuracy": 0.985 - }, - "0.01": { - "tp": 765, - "fn": 35, - "accuracy": 0.95625 - } - }, - "auroc": 0.9804145833333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9921968750000001 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9938687500000001 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9930328125000001 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9900875 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9866302083333334 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9883588541666666 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9911421874999999 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9902494791666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 780, - "fn": 20, - "accuracy": 0.975 - } - }, - "auroc": 0.9906958333333334 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9483208333333333 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9483208333333333 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.948065625 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.948065625 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - }, - "0.01": { - "tp": 345, - "fn": 55, - "accuracy": 0.8625 - } - }, - "auroc": 0.9481932291666666 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - }, - "0.01": { - "tp": 345, - "fn": 55, - "accuracy": 0.8625 - } - }, - "auroc": 0.9481932291666666 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 155, - "fn": 45, - "accuracy": 0.775 - } - }, - "auroc": 0.9251208333333334 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 155, - "fn": 45, - "accuracy": 0.775 - } - }, - "auroc": 0.9251208333333334 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 139, - "fn": 61, - "accuracy": 0.695 - } - }, - "auroc": 0.9033708333333332 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 139, - "fn": 61, - "accuracy": 0.695 - } - }, - "auroc": 0.9033708333333332 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - }, - "0.01": { - "tp": 294, - "fn": 106, - "accuracy": 0.735 - } - }, - "auroc": 0.9142458333333334 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - }, - "0.01": { - "tp": 294, - "fn": 106, - "accuracy": 0.735 - } - }, - "auroc": 0.9142458333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9982270833333333 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9982270833333333 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9938354166666667 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9938354166666667 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9960312499999999 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9960312499999999 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.995509375 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.995509375 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9829708333333333 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9829708333333333 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9892401041666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9892401041666667 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 156, - "fn": 44, - "accuracy": 0.78 - } - }, - "auroc": 0.9324166666666667 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 156, - "fn": 44, - "accuracy": 0.78 - } - }, - "auroc": 0.9324166666666667 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - }, - "0.01": { - "tp": 125, - "fn": 75, - "accuracy": 0.625 - } - }, - "auroc": 0.8881833333333334 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - }, - "0.01": { - "tp": 125, - "fn": 75, - "accuracy": 0.625 - } - }, - "auroc": 0.8881833333333334 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 348, - "fn": 52, - "accuracy": 0.87 - }, - "0.01": { - "tp": 281, - "fn": 119, - "accuracy": 0.7025 - } - }, - "auroc": 0.9103000000000001 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 348, - "fn": 52, - "accuracy": 0.87 - }, - "0.01": { - "tp": 281, - "fn": 119, - "accuracy": 0.7025 - } - }, - "auroc": 0.9103000000000001 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2132, - "fn": 68, - "accuracy": 0.9690909090909091 - }, - "0.01": { - "tp": 2059, - "fn": 141, - "accuracy": 0.9359090909090909 - } - }, - "auroc": 0.9774089015151515 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1188, - "fn": 12, - "accuracy": 0.99 - }, - "0.01": { - "tp": 1183, - "fn": 17, - "accuracy": 0.9858333333333333 - } - }, - "auroc": 0.9939487847222221 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3320, - "fn": 80, - "accuracy": 0.9764705882352941 - }, - "0.01": { - "tp": 3242, - "fn": 158, - "accuracy": 0.9535294117647058 - } - }, - "auroc": 0.9832465073529411 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2098, - "fn": 102, - "accuracy": 0.9536363636363636 - }, - "0.01": { - "tp": 1972, - "fn": 228, - "accuracy": 0.8963636363636364 - } - }, - "auroc": 0.961135321969697 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1177, - "fn": 23, - "accuracy": 0.9808333333333333 - }, - "0.01": { - "tp": 1154, - "fn": 46, - "accuracy": 0.9616666666666667 - } - }, - "auroc": 0.9816704861111111 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3275, - "fn": 125, - "accuracy": 0.9632352941176471 - }, - "0.01": { - "tp": 3126, - "fn": 274, - "accuracy": 0.9194117647058824 - } - }, - "auroc": 0.9683830269607843 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 4230, - "fn": 170, - "accuracy": 0.9613636363636363 - }, - "0.01": { - "tp": 4031, - "fn": 369, - "accuracy": 0.9161363636363636 - } - }, - "auroc": 0.9692721117424242 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2365, - "fn": 35, - "accuracy": 0.9854166666666667 - }, - "0.01": { - "tp": 2337, - "fn": 63, - "accuracy": 0.97375 - } - }, - "auroc": 0.9878096354166668 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 6595, - "fn": 205, - "accuracy": 0.9698529411764706 - }, - "0.01": { - "tp": 6368, - "fn": 432, - "accuracy": 0.9364705882352942 - } - }, - "auroc": 0.9758147671568627 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99775625 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9913989583333332 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9945776041666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9955510416666666 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9735375 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9845442708333332 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9966536458333334 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9824682291666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 778, - "fn": 22, - "accuracy": 0.9725 - } - }, - "auroc": 0.9895609375 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9629364583333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9868708333333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9749036458333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - }, - "0.01": { - "tp": 136, - "fn": 64, - "accuracy": 0.68 - } - }, - "auroc": 0.9082010416666666 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9684875000000001 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 324, - "fn": 76, - "accuracy": 0.81 - } - }, - "auroc": 0.9383442708333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - } - }, - "auroc": 0.93556875 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9776791666666667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 757, - "fn": 43, - "accuracy": 0.94625 - }, - "0.01": { - "tp": 698, - "fn": 102, - "accuracy": 0.8725 - } - }, - "auroc": 0.9566239583333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.9410864583333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.9423354166666666 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 319, - "fn": 81, - "accuracy": 0.7975 - } - }, - "auroc": 0.9417109375 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 144, - "fn": 56, - "accuracy": 0.72 - } - }, - "auroc": 0.9207010416666668 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 148, - "fn": 52, - "accuracy": 0.74 - } - }, - "auroc": 0.9270604166666667 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - }, - "0.01": { - "tp": 292, - "fn": 108, - "accuracy": 0.73 - } - }, - "auroc": 0.9238807291666667 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - }, - "0.01": { - "tp": 302, - "fn": 98, - "accuracy": 0.755 - } - }, - "auroc": 0.9308937500000001 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - }, - "0.01": { - "tp": 309, - "fn": 91, - "accuracy": 0.7725 - } - }, - "auroc": 0.9346979166666667 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 725, - "fn": 75, - "accuracy": 0.90625 - }, - "0.01": { - "tp": 611, - "fn": 189, - "accuracy": 0.76375 - } - }, - "auroc": 0.9327958333333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9377906250000001 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9882864583333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9630385416666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - }, - "0.01": { - "tp": 84, - "fn": 116, - "accuracy": 0.42 - } - }, - "auroc": 0.8688583333333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9457260416666666 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - }, - "0.01": { - "tp": 256, - "fn": 144, - "accuracy": 0.64 - } - }, - "auroc": 0.9072921875000001 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 359, - "fn": 41, - "accuracy": 0.8975 - }, - "0.01": { - "tp": 257, - "fn": 143, - "accuracy": 0.6425 - } - }, - "auroc": 0.9033244791666666 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9670062500000001 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 751, - "fn": 49, - "accuracy": 0.93875 - }, - "0.01": { - "tp": 628, - "fn": 172, - "accuracy": 0.785 - } - }, - "auroc": 0.9351653645833333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9881906250000001 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9828343750000001 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9855125 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 145, - "fn": 55, - "accuracy": 0.725 - }, - "0.01": { - "tp": 95, - "fn": 105, - "accuracy": 0.475 - } - }, - "auroc": 0.8532260416666666 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9598718749999999 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 337, - "fn": 63, - "accuracy": 0.8425 - }, - "0.01": { - "tp": 276, - "fn": 124, - "accuracy": 0.69 - } - }, - "auroc": 0.9065489583333335 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 345, - "fn": 55, - "accuracy": 0.8625 - }, - "0.01": { - "tp": 292, - "fn": 108, - "accuracy": 0.73 - } - }, - "auroc": 0.9207083333333334 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - } - }, - "auroc": 0.971353125 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 736, - "fn": 64, - "accuracy": 0.92 - }, - "0.01": { - "tp": 665, - "fn": 135, - "accuracy": 0.83125 - } - }, - "auroc": 0.9460307291666668 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9745218750000001 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9762864583333334 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - } - }, - "auroc": 0.9754041666666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9612031249999999 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9457760416666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 340, - "fn": 60, - "accuracy": 0.85 - } - }, - "auroc": 0.9534895833333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9678625000000002 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - } - }, - "auroc": 0.96103125 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 766, - "fn": 34, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 713, - "fn": 87, - "accuracy": 0.89125 - } - }, - "auroc": 0.964446875 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - }, - "0.01": { - "tp": 143, - "fn": 57, - "accuracy": 0.715 - } - }, - "auroc": 0.90899375 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - }, - "0.01": { - "tp": 143, - "fn": 57, - "accuracy": 0.715 - } - }, - "auroc": 0.90899375 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - }, - "0.01": { - "tp": 142, - "fn": 58, - "accuracy": 0.71 - } - }, - "auroc": 0.9070364583333335 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - }, - "0.01": { - "tp": 142, - "fn": 58, - "accuracy": 0.71 - } - }, - "auroc": 0.9070364583333335 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 330, - "fn": 70, - "accuracy": 0.825 - }, - "0.01": { - "tp": 285, - "fn": 115, - "accuracy": 0.7125 - } - }, - "auroc": 0.9080151041666666 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 330, - "fn": 70, - "accuracy": 0.825 - }, - "0.01": { - "tp": 285, - "fn": 115, - "accuracy": 0.7125 - } - }, - "auroc": 0.9080151041666666 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - }, - "0.01": { - "tp": 73, - "fn": 127, - "accuracy": 0.365 - } - }, - "auroc": 0.801990625 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 119, - "fn": 81, - "accuracy": 0.595 - }, - "0.01": { - "tp": 73, - "fn": 127, - "accuracy": 0.365 - } - }, - "auroc": 0.801990625 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 109, - "fn": 91, - "accuracy": 0.545 - }, - "0.01": { - "tp": 60, - "fn": 140, - "accuracy": 0.3 - } - }, - "auroc": 0.7614041666666667 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 109, - "fn": 91, - "accuracy": 0.545 - }, - "0.01": { - "tp": 60, - "fn": 140, - "accuracy": 0.3 - } - }, - "auroc": 0.7614041666666667 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 228, - "fn": 172, - "accuracy": 0.57 - }, - "0.01": { - "tp": 133, - "fn": 267, - "accuracy": 0.3325 - } - }, - "auroc": 0.7816973958333334 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 228, - "fn": 172, - "accuracy": 0.57 - }, - "0.01": { - "tp": 133, - "fn": 267, - "accuracy": 0.3325 - } - }, - "auroc": 0.7816973958333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9799249999999999 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9799249999999999 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.966209375 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.966209375 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9730671875000001 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9730671875000001 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.980515625 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.980515625 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 136, - "fn": 64, - "accuracy": 0.68 - } - }, - "auroc": 0.90950625 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 136, - "fn": 64, - "accuracy": 0.68 - } - }, - "auroc": 0.90950625 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - } - }, - "auroc": 0.9450109375 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - } - }, - "auroc": 0.9450109375 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 126, - "fn": 74, - "accuracy": 0.63 - }, - "0.01": { - "tp": 91, - "fn": 109, - "accuracy": 0.455 - } - }, - "auroc": 0.8166260416666666 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 126, - "fn": 74, - "accuracy": 0.63 - }, - "0.01": { - "tp": 91, - "fn": 109, - "accuracy": 0.455 - } - }, - "auroc": 0.8166260416666666 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 97, - "fn": 103, - "accuracy": 0.485 - }, - "0.01": { - "tp": 66, - "fn": 134, - "accuracy": 0.33 - } - }, - "auroc": 0.7801020833333334 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 97, - "fn": 103, - "accuracy": 0.485 - }, - "0.01": { - "tp": 66, - "fn": 134, - "accuracy": 0.33 - } - }, - "auroc": 0.7801020833333334 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 223, - "fn": 177, - "accuracy": 0.5575 - }, - "0.01": { - "tp": 157, - "fn": 243, - "accuracy": 0.3925 - } - }, - "auroc": 0.7983640624999999 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 223, - "fn": 177, - "accuracy": 0.5575 - }, - "0.01": { - "tp": 157, - "fn": 243, - "accuracy": 0.3925 - } - }, - "auroc": 0.7983640624999999 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1976, - "fn": 224, - "accuracy": 0.8981818181818182 - }, - "0.01": { - "tp": 1784, - "fn": 416, - "accuracy": 0.8109090909090909 - } - }, - "auroc": 0.9354848484848484 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1174, - "fn": 26, - "accuracy": 0.9783333333333334 - }, - "0.01": { - "tp": 1127, - "fn": 73, - "accuracy": 0.9391666666666667 - } - }, - "auroc": 0.9780020833333333 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3150, - "fn": 250, - "accuracy": 0.9264705882352942 - }, - "0.01": { - "tp": 2911, - "fn": 489, - "accuracy": 0.8561764705882353 - } - }, - "auroc": 0.950490931372549 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1784, - "fn": 416, - "accuracy": 0.8109090909090909 - }, - "0.01": { - "tp": 1412, - "fn": 788, - "accuracy": 0.6418181818181818 - } - }, - "auroc": 0.8938180871212121 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1141, - "fn": 59, - "accuracy": 0.9508333333333333 - }, - "0.01": { - "tp": 1040, - "fn": 160, - "accuracy": 0.8666666666666667 - } - }, - "auroc": 0.9534098958333332 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2925, - "fn": 475, - "accuracy": 0.8602941176470589 - }, - "0.01": { - "tp": 2452, - "fn": 948, - "accuracy": 0.7211764705882353 - } - }, - "auroc": 0.9148504901960783 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3760, - "fn": 640, - "accuracy": 0.8545454545454545 - }, - "0.01": { - "tp": 3196, - "fn": 1204, - "accuracy": 0.7263636363636363 - } - }, - "auroc": 0.9146514678030302 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2315, - "fn": 85, - "accuracy": 0.9645833333333333 - }, - "0.01": { - "tp": 2167, - "fn": 233, - "accuracy": 0.9029166666666667 - } - }, - "auroc": 0.9657059895833334 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 6075, - "fn": 725, - "accuracy": 0.8933823529411765 - }, - "0.01": { - "tp": 5363, - "fn": 1437, - "accuracy": 0.7886764705882353 - } - }, - "auroc": 0.9326707107843137 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99999375 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999940625 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999671875 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999114583333334 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9956156249999999 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9977635416666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999526041666666 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9977781250000001 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9988653645833334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9660552083333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9902833333333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9781692708333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.9582333333333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9735562500000001 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - } - }, - "auroc": 0.9658947916666667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9621442708333332 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9819197916666668 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 782, - "fn": 18, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 742, - "fn": 58, - "accuracy": 0.9275 - } - }, - "auroc": 0.9720320312500002 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9792364583333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9808958333333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9800661458333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9701989583333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9627427083333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - } - }, - "auroc": 0.9664708333333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9747177083333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9718192708333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 769, - "fn": 31, - "accuracy": 0.96125 - }, - "0.01": { - "tp": 742, - "fn": 58, - "accuracy": 0.9275 - } - }, - "auroc": 0.9732684895833333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.9395197916666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987020833333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9691109375 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9471729166666666 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9762125 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 359, - "fn": 41, - "accuracy": 0.8975 - } - }, - "auroc": 0.9616927083333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 340, - "fn": 60, - "accuracy": 0.85 - } - }, - "auroc": 0.9433463541666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9874572916666666 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 782, - "fn": 18, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 730, - "fn": 70, - "accuracy": 0.9125 - } - }, - "auroc": 0.9654018229166667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9891604166666668 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.994296875 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9917286458333334 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - }, - "0.01": { - "tp": 148, - "fn": 52, - "accuracy": 0.74 - } - }, - "auroc": 0.9219552083333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9766572916666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - }, - "0.01": { - "tp": 337, - "fn": 63, - "accuracy": 0.8425 - } - }, - "auroc": 0.94930625 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 344, - "fn": 56, - "accuracy": 0.86 - } - }, - "auroc": 0.9555578124999999 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9854770833333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 771, - "fn": 29, - "accuracy": 0.96375 - }, - "0.01": { - "tp": 731, - "fn": 69, - "accuracy": 0.91375 - } - }, - "auroc": 0.9705174479166667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9906843750000001 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9939635416666668 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9923239583333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9878375000000001 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9809479166666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9843927083333334 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9892609375000001 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9874557291666666 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 786, - "fn": 14, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 777, - "fn": 23, - "accuracy": 0.97125 - } - }, - "auroc": 0.9883583333333334 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - } - }, - "auroc": 0.9423239583333334 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - } - }, - "auroc": 0.9423239583333334 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.941034375 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.941034375 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - }, - "0.01": { - "tp": 330, - "fn": 70, - "accuracy": 0.825 - } - }, - "auroc": 0.9416791666666666 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - }, - "0.01": { - "tp": 330, - "fn": 70, - "accuracy": 0.825 - } - }, - "auroc": 0.9416791666666666 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - }, - "0.01": { - "tp": 143, - "fn": 57, - "accuracy": 0.715 - } - }, - "auroc": 0.8963375 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 163, - "fn": 37, - "accuracy": 0.815 - }, - "0.01": { - "tp": 143, - "fn": 57, - "accuracy": 0.715 - } - }, - "auroc": 0.8963375 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 155, - "fn": 45, - "accuracy": 0.775 - }, - "0.01": { - "tp": 128, - "fn": 72, - "accuracy": 0.64 - } - }, - "auroc": 0.868959375 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 155, - "fn": 45, - "accuracy": 0.775 - }, - "0.01": { - "tp": 128, - "fn": 72, - "accuracy": 0.64 - } - }, - "auroc": 0.868959375 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - }, - "0.01": { - "tp": 271, - "fn": 129, - "accuracy": 0.6775 - } - }, - "auroc": 0.8826484375 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - }, - "0.01": { - "tp": 271, - "fn": 129, - "accuracy": 0.6775 - } - }, - "auroc": 0.8826484375 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9977385416666666 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9977385416666666 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9919583333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9919583333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9948484375000001 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9948484375000001 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9936479166666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9936479166666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9744760416666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9744760416666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9840619791666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9840619791666667 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - }, - "0.01": { - "tp": 139, - "fn": 61, - "accuracy": 0.695 - } - }, - "auroc": 0.897634375 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - }, - "0.01": { - "tp": 139, - "fn": 61, - "accuracy": 0.695 - } - }, - "auroc": 0.897634375 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 139, - "fn": 61, - "accuracy": 0.695 - }, - "0.01": { - "tp": 116, - "fn": 84, - "accuracy": 0.58 - } - }, - "auroc": 0.8518270833333333 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 139, - "fn": 61, - "accuracy": 0.695 - }, - "0.01": { - "tp": 116, - "fn": 84, - "accuracy": 0.58 - } - }, - "auroc": 0.8518270833333333 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 301, - "fn": 99, - "accuracy": 0.7525 - }, - "0.01": { - "tp": 255, - "fn": 145, - "accuracy": 0.6375 - } - }, - "auroc": 0.8747307291666666 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 301, - "fn": 99, - "accuracy": 0.7525 - }, - "0.01": { - "tp": 255, - "fn": 145, - "accuracy": 0.6375 - } - }, - "auroc": 0.8747307291666666 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2085, - "fn": 115, - "accuracy": 0.9477272727272728 - }, - "0.01": { - "tp": 1979, - "fn": 221, - "accuracy": 0.8995454545454545 - } - }, - "auroc": 0.9629392992424243 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1189, - "fn": 11, - "accuracy": 0.9908333333333333 - }, - "0.01": { - "tp": 1180, - "fn": 20, - "accuracy": 0.9833333333333333 - } - }, - "auroc": 0.9930137152777778 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3274, - "fn": 126, - "accuracy": 0.9629411764705882 - }, - "0.01": { - "tp": 3159, - "fn": 241, - "accuracy": 0.9291176470588235 - } - }, - "auroc": 0.9735537990196079 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2011, - "fn": 189, - "accuracy": 0.9140909090909091 - }, - "0.01": { - "tp": 1854, - "fn": 346, - "accuracy": 0.8427272727272728 - } - }, - "auroc": 0.9466876893939393 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1169, - "fn": 31, - "accuracy": 0.9741666666666666 - }, - "0.01": { - "tp": 1137, - "fn": 63, - "accuracy": 0.9475 - } - }, - "auroc": 0.9776220486111111 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3180, - "fn": 220, - "accuracy": 0.9352941176470588 - }, - "0.01": { - "tp": 2991, - "fn": 409, - "accuracy": 0.8797058823529412 - } - }, - "auroc": 0.9576056985294118 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 4096, - "fn": 304, - "accuracy": 0.9309090909090909 - }, - "0.01": { - "tp": 3833, - "fn": 567, - "accuracy": 0.8711363636363636 - } - }, - "auroc": 0.9548134943181819 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2358, - "fn": 42, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 2317, - "fn": 83, - "accuracy": 0.9654166666666667 - } - }, - "auroc": 0.9853178819444444 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 6454, - "fn": 346, - "accuracy": 0.9491176470588235 - }, - "0.01": { - "tp": 6150, - "fn": 650, - "accuracy": 0.9044117647058824 - } - }, - "auroc": 0.9655797487745099 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99733125 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9888729166666665 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9931020833333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9960854166666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9772489583333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9866671874999999 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9967083333333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9830609375 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 781, - "fn": 19, - "accuracy": 0.97625 - } - }, - "auroc": 0.9898846354166667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9519875 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 140, - "fn": 60, - "accuracy": 0.7 - } - }, - "auroc": 0.9126604166666668 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - } - }, - "auroc": 0.9323239583333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9567354166666666 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 133, - "fn": 67, - "accuracy": 0.665 - }, - "0.01": { - "tp": 75, - "fn": 125, - "accuracy": 0.375 - } - }, - "auroc": 0.8313625000000001 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 325, - "fn": 75, - "accuracy": 0.8125 - }, - "0.01": { - "tp": 250, - "fn": 150, - "accuracy": 0.625 - } - }, - "auroc": 0.8940489583333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.9543614583333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 311, - "fn": 89, - "accuracy": 0.7775 - }, - "0.01": { - "tp": 215, - "fn": 185, - "accuracy": 0.5375 - } - }, - "auroc": 0.8720114583333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 698, - "fn": 102, - "accuracy": 0.8725 - }, - "0.01": { - "tp": 568, - "fn": 232, - "accuracy": 0.71 - } - }, - "auroc": 0.9131864583333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 153, - "fn": 47, - "accuracy": 0.765 - } - }, - "auroc": 0.94230625 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 156, - "fn": 44, - "accuracy": 0.78 - } - }, - "auroc": 0.9370208333333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 309, - "fn": 91, - "accuracy": 0.7725 - } - }, - "auroc": 0.9396635416666667 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 156, - "fn": 44, - "accuracy": 0.78 - } - }, - "auroc": 0.9379020833333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - }, - "0.01": { - "tp": 144, - "fn": 56, - "accuracy": 0.72 - } - }, - "auroc": 0.9179333333333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - }, - "0.01": { - "tp": 300, - "fn": 100, - "accuracy": 0.75 - } - }, - "auroc": 0.9279177083333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 309, - "fn": 91, - "accuracy": 0.7725 - } - }, - "auroc": 0.9401041666666666 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - }, - "0.01": { - "tp": 300, - "fn": 100, - "accuracy": 0.75 - } - }, - "auroc": 0.9274770833333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 732, - "fn": 68, - "accuracy": 0.915 - }, - "0.01": { - "tp": 609, - "fn": 191, - "accuracy": 0.76125 - } - }, - "auroc": 0.933790625 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.9247916666666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9901520833333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.957471875 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9436270833333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 125, - "fn": 75, - "accuracy": 0.625 - } - }, - "auroc": 0.9020614583333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 287, - "fn": 113, - "accuracy": 0.7175 - } - }, - "auroc": 0.9228442708333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 320, - "fn": 80, - "accuracy": 0.8 - } - }, - "auroc": 0.934209375 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 323, - "fn": 77, - "accuracy": 0.8075 - } - }, - "auroc": 0.9461067708333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - }, - "0.01": { - "tp": 643, - "fn": 157, - "accuracy": 0.80375 - } - }, - "auroc": 0.9401580729166666 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9628229166666666 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.9544604166666666 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 359, - "fn": 41, - "accuracy": 0.8975 - } - }, - "auroc": 0.9586416666666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - }, - "0.01": { - "tp": 140, - "fn": 60, - "accuracy": 0.7 - } - }, - "auroc": 0.9158239583333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 150, - "fn": 50, - "accuracy": 0.75 - }, - "0.01": { - "tp": 77, - "fn": 123, - "accuracy": 0.385 - } - }, - "auroc": 0.8425010416666666 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 324, - "fn": 76, - "accuracy": 0.81 - }, - "0.01": { - "tp": 217, - "fn": 183, - "accuracy": 0.5425 - } - }, - "auroc": 0.8791625000000001 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - }, - "0.01": { - "tp": 325, - "fn": 75, - "accuracy": 0.8125 - } - }, - "auroc": 0.9393234375 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 343, - "fn": 57, - "accuracy": 0.8575 - }, - "0.01": { - "tp": 251, - "fn": 149, - "accuracy": 0.6275 - } - }, - "auroc": 0.8984807291666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 715, - "fn": 85, - "accuracy": 0.89375 - }, - "0.01": { - "tp": 576, - "fn": 224, - "accuracy": 0.72 - } - }, - "auroc": 0.9189020833333332 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9804854166666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9824020833333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9814437500000001 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9698416666666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - } - }, - "auroc": 0.9473552083333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 345, - "fn": 55, - "accuracy": 0.8625 - } - }, - "auroc": 0.9585984375000001 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9751635416666666 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9648786458333334 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 723, - "fn": 77, - "accuracy": 0.90375 - } - }, - "auroc": 0.97002109375 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 145, - "fn": 55, - "accuracy": 0.725 - } - }, - "auroc": 0.92035625 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 145, - "fn": 55, - "accuracy": 0.725 - } - }, - "auroc": 0.92035625 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - }, - "0.01": { - "tp": 152, - "fn": 48, - "accuracy": 0.76 - } - }, - "auroc": 0.9250770833333333 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - }, - "0.01": { - "tp": 152, - "fn": 48, - "accuracy": 0.76 - } - }, - "auroc": 0.9250770833333333 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - }, - "0.01": { - "tp": 297, - "fn": 103, - "accuracy": 0.7425 - } - }, - "auroc": 0.9227166666666666 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - }, - "0.01": { - "tp": 297, - "fn": 103, - "accuracy": 0.7425 - } - }, - "auroc": 0.9227166666666666 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - }, - "0.01": { - "tp": 137, - "fn": 63, - "accuracy": 0.685 - } - }, - "auroc": 0.9045416666666666 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - }, - "0.01": { - "tp": 137, - "fn": 63, - "accuracy": 0.685 - } - }, - "auroc": 0.9045416666666666 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - }, - "0.01": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - } - }, - "auroc": 0.8967135416666667 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - }, - "0.01": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - } - }, - "auroc": 0.8967135416666667 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 331, - "fn": 69, - "accuracy": 0.8275 - }, - "0.01": { - "tp": 267, - "fn": 133, - "accuracy": 0.6675 - } - }, - "auroc": 0.9006276041666668 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 331, - "fn": 69, - "accuracy": 0.8275 - }, - "0.01": { - "tp": 267, - "fn": 133, - "accuracy": 0.6675 - } - }, - "auroc": 0.9006276041666668 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9874781250000001 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9874781250000001 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9856343750000001 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9856343750000001 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.98655625 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.98655625 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9940760416666666 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9940760416666666 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.974459375 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.974459375 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9842677083333333 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9842677083333333 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - }, - "0.01": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - } - }, - "auroc": 0.896225 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - }, - "0.01": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - } - }, - "auroc": 0.896225 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 157, - "fn": 43, - "accuracy": 0.785 - }, - "0.01": { - "tp": 121, - "fn": 79, - "accuracy": 0.605 - } - }, - "auroc": 0.8808020833333334 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 157, - "fn": 43, - "accuracy": 0.785 - }, - "0.01": { - "tp": 121, - "fn": 79, - "accuracy": 0.605 - } - }, - "auroc": 0.8808020833333334 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 324, - "fn": 76, - "accuracy": 0.81 - }, - "0.01": { - "tp": 255, - "fn": 145, - "accuracy": 0.6375 - } - }, - "auroc": 0.8885135416666667 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 324, - "fn": 76, - "accuracy": 0.81 - }, - "0.01": { - "tp": 255, - "fn": 145, - "accuracy": 0.6375 - } - }, - "auroc": 0.8885135416666667 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2079, - "fn": 121, - "accuracy": 0.945 - }, - "0.01": { - "tp": 1869, - "fn": 331, - "accuracy": 0.8495454545454545 - } - }, - "auroc": 0.951127462121212 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1150, - "fn": 50, - "accuracy": 0.9583333333333334 - }, - "0.01": { - "tp": 1049, - "fn": 151, - "accuracy": 0.8741666666666666 - } - }, - "auroc": 0.960928125 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3229, - "fn": 171, - "accuracy": 0.9497058823529412 - }, - "0.01": { - "tp": 2918, - "fn": 482, - "accuracy": 0.8582352941176471 - } - }, - "auroc": 0.9545865196078431 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2008, - "fn": 192, - "accuracy": 0.9127272727272727 - }, - "0.01": { - "tp": 1798, - "fn": 402, - "accuracy": 0.8172727272727273 - } - }, - "auroc": 0.9438820075757577 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1031, - "fn": 169, - "accuracy": 0.8591666666666666 - }, - "0.01": { - "tp": 775, - "fn": 425, - "accuracy": 0.6458333333333334 - } - }, - "auroc": 0.9030770833333334 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3039, - "fn": 361, - "accuracy": 0.8938235294117647 - }, - "0.01": { - "tp": 2573, - "fn": 827, - "accuracy": 0.7567647058823529 - } - }, - "auroc": 0.929480269607843 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 4087, - "fn": 313, - "accuracy": 0.9288636363636363 - }, - "0.01": { - "tp": 3667, - "fn": 733, - "accuracy": 0.8334090909090909 - } - }, - "auroc": 0.9475047348484849 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2181, - "fn": 219, - "accuracy": 0.90875 - }, - "0.01": { - "tp": 1824, - "fn": 576, - "accuracy": 0.76 - } - }, - "auroc": 0.9320026041666667 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 6268, - "fn": 532, - "accuracy": 0.9217647058823529 - }, - "0.01": { - "tp": 5491, - "fn": 1309, - "accuracy": 0.8075 - } - }, - "auroc": 0.9420333946078432 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999385416666666 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994739583333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99970625 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997927083333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9906822916666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9952375 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999865625 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.995078125 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - } - }, - "auroc": 0.997471875 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9670208333333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9896072916666666 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9783140625 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.9230729166666667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.973271875 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 337, - "fn": 63, - "accuracy": 0.8425 - } - }, - "auroc": 0.9481723958333332 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 336, - "fn": 64, - "accuracy": 0.84 - } - }, - "auroc": 0.945046875 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9814395833333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 773, - "fn": 27, - "accuracy": 0.96625 - }, - "0.01": { - "tp": 721, - "fn": 79, - "accuracy": 0.90125 - } - }, - "auroc": 0.9632432291666666 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9792020833333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9815895833333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9803958333333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.97014375 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.96133125 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.9657375000000001 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9746729166666667 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9714604166666667 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 736, - "fn": 64, - "accuracy": 0.92 - } - }, - "auroc": 0.9730666666666665 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9397916666666666 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998253125 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9690223958333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 128, - "fn": 72, - "accuracy": 0.64 - } - }, - "auroc": 0.9095291666666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9623614583333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 315, - "fn": 85, - "accuracy": 0.7875 - } - }, - "auroc": 0.9359453124999999 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 303, - "fn": 97, - "accuracy": 0.7575 - } - }, - "auroc": 0.9246604166666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9803072916666666 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 773, - "fn": 27, - "accuracy": 0.96625 - }, - "0.01": { - "tp": 690, - "fn": 110, - "accuracy": 0.8625 - } - }, - "auroc": 0.9524838541666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.989284375 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9924989583333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9908916666666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 156, - "fn": 44, - "accuracy": 0.78 - }, - "0.01": { - "tp": 114, - "fn": 86, - "accuracy": 0.57 - } - }, - "auroc": 0.8732666666666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9719729166666666 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - }, - "0.01": { - "tp": 303, - "fn": 97, - "accuracy": 0.7575 - } - }, - "auroc": 0.9226197916666666 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - }, - "0.01": { - "tp": 310, - "fn": 90, - "accuracy": 0.775 - } - }, - "auroc": 0.9312755208333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9822359375 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 749, - "fn": 51, - "accuracy": 0.93625 - }, - "0.01": { - "tp": 697, - "fn": 103, - "accuracy": 0.87125 - } - }, - "auroc": 0.9567557291666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9912635416666669 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9937760416666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9925197916666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9871708333333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.976465625 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9818182291666666 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9892171875 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9851208333333332 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 786, - "fn": 14, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 775, - "fn": 25, - "accuracy": 0.96875 - } - }, - "auroc": 0.9871690104166666 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9431364583333333 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9431364583333333 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9425270833333332 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9425270833333332 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9428317708333334 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9428317708333334 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 110, - "fn": 90, - "accuracy": 0.55 - } - }, - "auroc": 0.8606229166666666 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 110, - "fn": 90, - "accuracy": 0.55 - } - }, - "auroc": 0.8606229166666666 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 135, - "fn": 65, - "accuracy": 0.675 - }, - "0.01": { - "tp": 100, - "fn": 100, - "accuracy": 0.5 - } - }, - "auroc": 0.8171145833333333 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 135, - "fn": 65, - "accuracy": 0.675 - }, - "0.01": { - "tp": 100, - "fn": 100, - "accuracy": 0.5 - } - }, - "auroc": 0.8171145833333333 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 286, - "fn": 114, - "accuracy": 0.715 - }, - "0.01": { - "tp": 210, - "fn": 190, - "accuracy": 0.525 - } - }, - "auroc": 0.83886875 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 286, - "fn": 114, - "accuracy": 0.715 - }, - "0.01": { - "tp": 210, - "fn": 190, - "accuracy": 0.525 - } - }, - "auroc": 0.83886875 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9976958333333333 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9976958333333333 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9921958333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9921958333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9949458333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9949458333333334 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9927104166666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9927104166666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9548875 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9548875 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9737989583333334 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9737989583333334 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 150, - "fn": 50, - "accuracy": 0.75 - }, - "0.01": { - "tp": 124, - "fn": 76, - "accuracy": 0.62 - } - }, - "auroc": 0.8730885416666667 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 150, - "fn": 50, - "accuracy": 0.75 - }, - "0.01": { - "tp": 124, - "fn": 76, - "accuracy": 0.62 - } - }, - "auroc": 0.8730885416666667 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 132, - "fn": 68, - "accuracy": 0.66 - }, - "0.01": { - "tp": 105, - "fn": 95, - "accuracy": 0.525 - } - }, - "auroc": 0.8301958333333334 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 132, - "fn": 68, - "accuracy": 0.66 - }, - "0.01": { - "tp": 105, - "fn": 95, - "accuracy": 0.525 - } - }, - "auroc": 0.8301958333333334 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 282, - "fn": 118, - "accuracy": 0.705 - }, - "0.01": { - "tp": 229, - "fn": 171, - "accuracy": 0.5725 - } - }, - "auroc": 0.8516421875 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 282, - "fn": 118, - "accuracy": 0.705 - }, - "0.01": { - "tp": 229, - "fn": 171, - "accuracy": 0.5725 - } - }, - "auroc": 0.8516421875 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2062, - "fn": 138, - "accuracy": 0.9372727272727273 - }, - "0.01": { - "tp": 1940, - "fn": 260, - "accuracy": 0.8818181818181818 - } - }, - "auroc": 0.9576141098484848 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1189, - "fn": 11, - "accuracy": 0.9908333333333333 - }, - "0.01": { - "tp": 1179, - "fn": 21, - "accuracy": 0.9825 - } - }, - "auroc": 0.9925331597222222 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3251, - "fn": 149, - "accuracy": 0.9561764705882353 - }, - "0.01": { - "tp": 3119, - "fn": 281, - "accuracy": 0.9173529411764706 - } - }, - "auroc": 0.9699384803921569 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1942, - "fn": 258, - "accuracy": 0.8827272727272727 - }, - "0.01": { - "tp": 1701, - "fn": 499, - "accuracy": 0.7731818181818182 - } - }, - "auroc": 0.9272633522727274 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1167, - "fn": 33, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 1127, - "fn": 73, - "accuracy": 0.9391666666666667 - } - }, - "auroc": 0.9726809027777779 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3109, - "fn": 291, - "accuracy": 0.9144117647058824 - }, - "0.01": { - "tp": 2828, - "fn": 572, - "accuracy": 0.831764705882353 - } - }, - "auroc": 0.9432930759803921 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 4004, - "fn": 396, - "accuracy": 0.91 - }, - "0.01": { - "tp": 3641, - "fn": 759, - "accuracy": 0.8275 - } - }, - "auroc": 0.9424387310606062 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2356, - "fn": 44, - "accuracy": 0.9816666666666667 - }, - "0.01": { - "tp": 2306, - "fn": 94, - "accuracy": 0.9608333333333333 - } - }, - "auroc": 0.98260703125 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 6360, - "fn": 440, - "accuracy": 0.9352941176470588 - }, - "0.01": { - "tp": 5947, - "fn": 853, - "accuracy": 0.8745588235294117 - } - }, - "auroc": 0.9566157781862744 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99994375 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994895833333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997166666666666 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99983125 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9907291666666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9952802083333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998874999999999 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.995109375 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - } - }, - "auroc": 0.9974984375 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9656270833333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9894458333333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9775364583333335 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 144, - "fn": 56, - "accuracy": 0.72 - } - }, - "auroc": 0.9192114583333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.973453125 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 334, - "fn": 66, - "accuracy": 0.835 - } - }, - "auroc": 0.9463322916666668 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 330, - "fn": 70, - "accuracy": 0.825 - } - }, - "auroc": 0.9424192708333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9814494791666667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 714, - "fn": 86, - "accuracy": 0.8925 - } - }, - "auroc": 0.961934375 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9791083333333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9811583333333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9801333333333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9698885416666667 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9611552083333332 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 359, - "fn": 41, - "accuracy": 0.8975 - } - }, - "auroc": 0.965521875 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9744984375 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9711567708333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 734, - "fn": 66, - "accuracy": 0.9175 - } - }, - "auroc": 0.9728276041666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9403375 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981729166666666 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9692552083333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 123, - "fn": 77, - "accuracy": 0.615 - } - }, - "auroc": 0.9036416666666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9601239583333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 309, - "fn": 91, - "accuracy": 0.7725 - } - }, - "auroc": 0.9318828124999999 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 301, - "fn": 99, - "accuracy": 0.7525 - } - }, - "auroc": 0.9219895833333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9791484375 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 687, - "fn": 113, - "accuracy": 0.85875 - } - }, - "auroc": 0.9505690104166667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9894062499999999 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9922458333333335 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9908260416666668 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 155, - "fn": 45, - "accuracy": 0.775 - }, - "0.01": { - "tp": 107, - "fn": 93, - "accuracy": 0.535 - } - }, - "auroc": 0.8682489583333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9705364583333334 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - }, - "0.01": { - "tp": 296, - "fn": 104, - "accuracy": 0.74 - } - }, - "auroc": 0.9193927083333334 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - }, - "0.01": { - "tp": 304, - "fn": 96, - "accuracy": 0.76 - } - }, - "auroc": 0.9288276041666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9813911458333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 749, - "fn": 51, - "accuracy": 0.93625 - }, - "0.01": { - "tp": 691, - "fn": 109, - "accuracy": 0.86375 - } - }, - "auroc": 0.955109375 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9913031250000001 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9941729166666666 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9927380208333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9873385416666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9770197916666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9821791666666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9893208333333334 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9855963541666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 785, - "fn": 15, - "accuracy": 0.98125 - }, - "0.01": { - "tp": 773, - "fn": 27, - "accuracy": 0.96625 - } - }, - "auroc": 0.98745859375 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9429791666666666 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9429791666666666 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.942409375 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.942409375 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9426942708333335 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9426942708333335 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 109, - "fn": 91, - "accuracy": 0.545 - } - }, - "auroc": 0.8579854166666666 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 109, - "fn": 91, - "accuracy": 0.545 - } - }, - "auroc": 0.8579854166666666 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 133, - "fn": 67, - "accuracy": 0.665 - }, - "0.01": { - "tp": 98, - "fn": 102, - "accuracy": 0.49 - } - }, - "auroc": 0.814759375 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 133, - "fn": 67, - "accuracy": 0.665 - }, - "0.01": { - "tp": 98, - "fn": 102, - "accuracy": 0.49 - } - }, - "auroc": 0.814759375 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 284, - "fn": 116, - "accuracy": 0.71 - }, - "0.01": { - "tp": 207, - "fn": 193, - "accuracy": 0.5175 - } - }, - "auroc": 0.8363723958333333 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 284, - "fn": 116, - "accuracy": 0.71 - }, - "0.01": { - "tp": 207, - "fn": 193, - "accuracy": 0.5175 - } - }, - "auroc": 0.8363723958333333 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.997721875 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.997721875 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9921531250000001 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9921531250000001 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9949375 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9949375 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9927197916666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9927197916666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.9541229166666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.9541229166666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9734213541666666 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9734213541666666 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 124, - "fn": 76, - "accuracy": 0.62 - } - }, - "auroc": 0.8739104166666667 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 124, - "fn": 76, - "accuracy": 0.62 - } - }, - "auroc": 0.8739104166666667 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - }, - "0.01": { - "tp": 104, - "fn": 96, - "accuracy": 0.52 - } - }, - "auroc": 0.8263854166666667 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - }, - "0.01": { - "tp": 104, - "fn": 96, - "accuracy": 0.52 - } - }, - "auroc": 0.8263854166666667 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 281, - "fn": 119, - "accuracy": 0.7025 - }, - "0.01": { - "tp": 228, - "fn": 172, - "accuracy": 0.57 - } - }, - "auroc": 0.8501479166666666 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 281, - "fn": 119, - "accuracy": 0.7025 - }, - "0.01": { - "tp": 228, - "fn": 172, - "accuracy": 0.57 - } - }, - "auroc": 0.8501479166666666 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2063, - "fn": 137, - "accuracy": 0.9377272727272727 - }, - "0.01": { - "tp": 1940, - "fn": 260, - "accuracy": 0.8818181818181818 - } - }, - "auroc": 0.957367518939394 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1189, - "fn": 11, - "accuracy": 0.9908333333333333 - }, - "0.01": { - "tp": 1177, - "fn": 23, - "accuracy": 0.9808333333333333 - } - }, - "auroc": 0.9924475694444443 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3252, - "fn": 148, - "accuracy": 0.9564705882352941 - }, - "0.01": { - "tp": 3117, - "fn": 283, - "accuracy": 0.9167647058823529 - } - }, - "auroc": 0.9697487132352942 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1932, - "fn": 268, - "accuracy": 0.8781818181818182 - }, - "0.01": { - "tp": 1681, - "fn": 519, - "accuracy": 0.764090909090909 - } - }, - "auroc": 0.925271875 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1166, - "fn": 34, - "accuracy": 0.9716666666666667 - }, - "0.01": { - "tp": 1124, - "fn": 76, - "accuracy": 0.9366666666666666 - } - }, - "auroc": 0.9721696180555556 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3098, - "fn": 302, - "accuracy": 0.9111764705882353 - }, - "0.01": { - "tp": 2805, - "fn": 595, - "accuracy": 0.825 - } - }, - "auroc": 0.9418240196078431 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3995, - "fn": 405, - "accuracy": 0.9079545454545455 - }, - "0.01": { - "tp": 3621, - "fn": 779, - "accuracy": 0.8229545454545455 - } - }, - "auroc": 0.9413196969696971 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2355, - "fn": 45, - "accuracy": 0.98125 - }, - "0.01": { - "tp": 2301, - "fn": 99, - "accuracy": 0.95875 - } - }, - "auroc": 0.98230859375 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 6350, - "fn": 450, - "accuracy": 0.9338235294117647 - }, - "0.01": { - "tp": 5922, - "fn": 878, - "accuracy": 0.8708823529411764 - } - }, - "auroc": 0.9557863664215686 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9772489583333334 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9632895833333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9702692708333334 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9762197916666666 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.967190625 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9717052083333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9767343749999999 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9652401041666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 788, - "fn": 12, - "accuracy": 0.985 - } - }, - "auroc": 0.9709872395833333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9769322916666666 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.98723125 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9820817708333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9882833333333332 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9748416666666668 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9815625000000001 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9826078125 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9810364583333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 789, - "fn": 11, - "accuracy": 0.98625 - }, - "0.01": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - } - }, - "auroc": 0.9818221354166666 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9805572916666666 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.973371875 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9769645833333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9754552083333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9705781250000001 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9730166666666669 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9780062500000001 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9719750000000001 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 784, - "fn": 16, - "accuracy": 0.98 - }, - "0.01": { - "tp": 769, - "fn": 31, - "accuracy": 0.96125 - } - }, - "auroc": 0.974990625 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9431114583333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.995890625 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9695010416666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9934781250000001 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9863614583333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9899197916666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9682947916666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9911260416666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 763, - "fn": 37, - "accuracy": 0.95375 - } - }, - "auroc": 0.9797104166666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9932489583333334 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9843572916666666 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.988803125 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9863083333333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9877499999999999 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9870291666666666 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9897786458333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9860536458333334 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - } - }, - "auroc": 0.9879161458333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.980890625 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9698354166666665 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9753630208333334 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9742083333333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9767937499999999 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9755010416666666 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9775494791666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9733145833333334 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 779, - "fn": 21, - "accuracy": 0.97375 - } - }, - "auroc": 0.97543203125 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9579374999999999 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9579374999999999 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9599458333333333 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9599458333333333 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9589416666666668 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9589416666666668 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9681968750000001 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9681968750000001 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9601510416666666 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9601510416666666 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9641739583333333 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9641739583333333 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9615354166666668 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9615354166666668 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9597833333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9597833333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9606593750000001 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9606593750000001 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.96516875 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.96516875 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9688281249999999 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9688281249999999 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9669984375 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9669984375 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9670958333333333 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9670958333333333 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9480375 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9480375 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - } - }, - "auroc": 0.9575666666666667 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - } - }, - "auroc": 0.9575666666666667 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2156, - "fn": 44, - "accuracy": 0.98 - }, - "0.01": { - "tp": 2097, - "fn": 103, - "accuracy": 0.9531818181818181 - } - }, - "auroc": 0.9701749053030303 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1191, - "fn": 9, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 1176, - "fn": 24, - "accuracy": 0.98 - } - }, - "auroc": 0.9789960069444446 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3347, - "fn": 53, - "accuracy": 0.9844117647058823 - }, - "0.01": { - "tp": 3273, - "fn": 127, - "accuracy": 0.9626470588235294 - } - }, - "auroc": 0.9732882352941177 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2151, - "fn": 49, - "accuracy": 0.9777272727272728 - }, - "0.01": { - "tp": 2106, - "fn": 94, - "accuracy": 0.9572727272727273 - } - }, - "auroc": 0.9718817234848485 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1187, - "fn": 13, - "accuracy": 0.9891666666666666 - }, - "0.01": { - "tp": 1166, - "fn": 34, - "accuracy": 0.9716666666666667 - } - }, - "auroc": 0.9772526041666667 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3338, - "fn": 62, - "accuracy": 0.981764705882353 - }, - "0.01": { - "tp": 3272, - "fn": 128, - "accuracy": 0.9623529411764706 - } - }, - "auroc": 0.9737773284313724 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 4307, - "fn": 93, - "accuracy": 0.9788636363636364 - }, - "0.01": { - "tp": 4203, - "fn": 197, - "accuracy": 0.9552272727272727 - } - }, - "auroc": 0.9710283143939393 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2378, - "fn": 22, - "accuracy": 0.9908333333333333 - }, - "0.01": { - "tp": 2342, - "fn": 58, - "accuracy": 0.9758333333333333 - } - }, - "auroc": 0.9781243055555555 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 6685, - "fn": 115, - "accuracy": 0.9830882352941176 - }, - "0.01": { - "tp": 6545, - "fn": 255, - "accuracy": 0.9625 - } - }, - "auroc": 0.9735327818627451 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99305 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.993671875 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9933609375000001 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9921062500000001 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9848197916666668 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9884630208333334 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9925781250000001 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9892458333333334 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - } - }, - "auroc": 0.9909119791666667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9483864583333332 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9894302083333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9689083333333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - } - }, - "auroc": 0.92818125 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9737885416666667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9509848958333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 339, - "fn": 61, - "accuracy": 0.8475 - } - }, - "auroc": 0.9382838541666667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.981609375 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 782, - "fn": 18, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 724, - "fn": 76, - "accuracy": 0.905 - } - }, - "auroc": 0.9599466145833334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9674114583333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9766083333333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9720098958333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9541416666666667 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9598322916666666 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9569869791666668 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 359, - "fn": 41, - "accuracy": 0.8975 - } - }, - "auroc": 0.9607765625 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - } - }, - "auroc": 0.9682203125000001 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 766, - "fn": 34, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 722, - "fn": 78, - "accuracy": 0.9025 - } - }, - "auroc": 0.9644984375 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 156, - "fn": 44, - "accuracy": 0.78 - } - }, - "auroc": 0.9228791666666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9919291666666666 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9574041666666666 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9163770833333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9736208333333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - } - }, - "auroc": 0.9449989583333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 325, - "fn": 75, - "accuracy": 0.8125 - } - }, - "auroc": 0.919628125 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9827750000000001 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 787, - "fn": 13, - "accuracy": 0.98375 - }, - "0.01": { - "tp": 717, - "fn": 83, - "accuracy": 0.89625 - } - }, - "auroc": 0.9512015625 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9834354166666668 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9916302083333334 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9875328125 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 129, - "fn": 71, - "accuracy": 0.645 - } - }, - "auroc": 0.8871020833333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9747635416666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - } - }, - "auroc": 0.9309328125 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 321, - "fn": 79, - "accuracy": 0.8025 - } - }, - "auroc": 0.9352687499999999 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9831968750000001 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 772, - "fn": 28, - "accuracy": 0.965 - }, - "0.01": { - "tp": 706, - "fn": 94, - "accuracy": 0.8825 - } - }, - "auroc": 0.9592328125 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.984253125 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9902270833333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9872401041666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9763593749999999 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9747395833333332 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9755494791666668 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.98030625 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9824833333333334 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 785, - "fn": 15, - "accuracy": 0.98125 - }, - "0.01": { - "tp": 769, - "fn": 31, - "accuracy": 0.96125 - } - }, - "auroc": 0.9813947916666668 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.9257364583333334 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.9257364583333334 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 153, - "fn": 47, - "accuracy": 0.765 - } - }, - "auroc": 0.9218385416666666 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 153, - "fn": 47, - "accuracy": 0.765 - } - }, - "auroc": 0.9218385416666666 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 314, - "fn": 86, - "accuracy": 0.785 - } - }, - "auroc": 0.9237875 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 314, - "fn": 86, - "accuracy": 0.785 - } - }, - "auroc": 0.9237875 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 160, - "fn": 40, - "accuracy": 0.8 - }, - "0.01": { - "tp": 109, - "fn": 91, - "accuracy": 0.545 - } - }, - "auroc": 0.8531562500000001 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 160, - "fn": 40, - "accuracy": 0.8 - }, - "0.01": { - "tp": 109, - "fn": 91, - "accuracy": 0.545 - } - }, - "auroc": 0.8531562500000001 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - }, - "0.01": { - "tp": 93, - "fn": 107, - "accuracy": 0.465 - } - }, - "auroc": 0.8310375000000001 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - }, - "0.01": { - "tp": 93, - "fn": 107, - "accuracy": 0.465 - } - }, - "auroc": 0.8310375000000001 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - }, - "0.01": { - "tp": 202, - "fn": 198, - "accuracy": 0.505 - } - }, - "auroc": 0.842096875 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - }, - "0.01": { - "tp": 202, - "fn": 198, - "accuracy": 0.505 - } - }, - "auroc": 0.842096875 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.991546875 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.991546875 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9845645833333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9845645833333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9880557291666667 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9880557291666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9792677083333333 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9792677083333333 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9364708333333334 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9364708333333334 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9578692708333333 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9578692708333333 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 157, - "fn": 43, - "accuracy": 0.785 - }, - "0.01": { - "tp": 120, - "fn": 80, - "accuracy": 0.6 - } - }, - "auroc": 0.8658760416666667 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 157, - "fn": 43, - "accuracy": 0.785 - }, - "0.01": { - "tp": 120, - "fn": 80, - "accuracy": 0.6 - } - }, - "auroc": 0.8658760416666667 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 140, - "fn": 60, - "accuracy": 0.7 - }, - "0.01": { - "tp": 104, - "fn": 96, - "accuracy": 0.52 - } - }, - "auroc": 0.82620625 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 140, - "fn": 60, - "accuracy": 0.7 - }, - "0.01": { - "tp": 104, - "fn": 96, - "accuracy": 0.52 - } - }, - "auroc": 0.82620625 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 297, - "fn": 103, - "accuracy": 0.7425 - }, - "0.01": { - "tp": 224, - "fn": 176, - "accuracy": 0.56 - } - }, - "auroc": 0.8460411458333333 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 297, - "fn": 103, - "accuracy": 0.7425 - }, - "0.01": { - "tp": 224, - "fn": 176, - "accuracy": 0.56 - } - }, - "auroc": 0.8460411458333333 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2074, - "fn": 126, - "accuracy": 0.9427272727272727 - }, - "0.01": { - "tp": 1881, - "fn": 319, - "accuracy": 0.855 - } - }, - "auroc": 0.9468180871212122 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1187, - "fn": 13, - "accuracy": 0.9891666666666666 - }, - "0.01": { - "tp": 1172, - "fn": 28, - "accuracy": 0.9766666666666667 - } - }, - "auroc": 0.9889161458333333 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3261, - "fn": 139, - "accuracy": 0.9591176470588235 - }, - "0.01": { - "tp": 3053, - "fn": 347, - "accuracy": 0.8979411764705882 - } - }, - "auroc": 0.9616762254901962 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2021, - "fn": 179, - "accuracy": 0.9186363636363636 - }, - "0.01": { - "tp": 1750, - "fn": 450, - "accuracy": 0.7954545454545454 - } - }, - "auroc": 0.923125946969697 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1169, - "fn": 31, - "accuracy": 0.9741666666666666 - }, - "0.01": { - "tp": 1134, - "fn": 66, - "accuracy": 0.945 - } - }, - "auroc": 0.9735940972222222 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 210, - "accuracy": 0.9382352941176471 - }, - "0.01": { - "tp": 2884, - "fn": 516, - "accuracy": 0.8482352941176471 - } - }, - "auroc": 0.9409382352941177 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 4095, - "fn": 305, - "accuracy": 0.9306818181818182 - }, - "0.01": { - "tp": 3631, - "fn": 769, - "accuracy": 0.8252272727272727 - } - }, - "auroc": 0.9349720170454545 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2356, - "fn": 44, - "accuracy": 0.9816666666666667 - }, - "0.01": { - "tp": 2306, - "fn": 94, - "accuracy": 0.9608333333333333 - } - }, - "auroc": 0.9812551215277778 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 6451, - "fn": 349, - "accuracy": 0.9486764705882353 - }, - "0.01": { - "tp": 5937, - "fn": 863, - "accuracy": 0.8730882352941176 - } - }, - "auroc": 0.9513072303921568 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999760416666666 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998791666666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999276041666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998260416666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9935645833333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9966953125 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999010416666666 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.996721875 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9983114583333335 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9662302083333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9896270833333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9779286458333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 159, - "fn": 41, - "accuracy": 0.795 - } - }, - "auroc": 0.9408239583333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9760104166666667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9584171875 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 347, - "fn": 53, - "accuracy": 0.8675 - } - }, - "auroc": 0.9535270833333335 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9828187500000001 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 778, - "fn": 22, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 733, - "fn": 67, - "accuracy": 0.91625 - } - }, - "auroc": 0.9681729166666666 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9767802083333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9826375 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9797088541666668 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9702416666666667 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.9612583333333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.96575 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9735109375 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - } - }, - "auroc": 0.9719479166666667 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 767, - "fn": 33, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 733, - "fn": 67, - "accuracy": 0.91625 - } - }, - "auroc": 0.9727294270833333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9403125 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998453125 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9693828124999999 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 155, - "fn": 45, - "accuracy": 0.775 - } - }, - "auroc": 0.9339333333333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.975 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 347, - "fn": 53, - "accuracy": 0.8675 - } - }, - "auroc": 0.9544666666666666 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 331, - "fn": 69, - "accuracy": 0.8275 - } - }, - "auroc": 0.9371229166666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9867265624999999 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 775, - "fn": 25, - "accuracy": 0.96875 - }, - "0.01": { - "tp": 723, - "fn": 77, - "accuracy": 0.90375 - } - }, - "auroc": 0.9619247395833334 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9898562500000001 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9936104166666666 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9917333333333335 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - }, - "0.01": { - "tp": 131, - "fn": 69, - "accuracy": 0.655 - } - }, - "auroc": 0.8958760416666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9785906249999999 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 323, - "fn": 77, - "accuracy": 0.8075 - } - }, - "auroc": 0.9372333333333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - }, - "0.01": { - "tp": 328, - "fn": 72, - "accuracy": 0.82 - } - }, - "auroc": 0.9428661458333334 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9861005208333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 763, - "fn": 37, - "accuracy": 0.95375 - }, - "0.01": { - "tp": 718, - "fn": 82, - "accuracy": 0.8975 - } - }, - "auroc": 0.9644833333333334 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9907614583333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9953708333333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9930661458333334 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9879635416666668 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.979753125 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9838583333333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9893624999999999 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9875619791666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 787, - "fn": 13, - "accuracy": 0.98375 - }, - "0.01": { - "tp": 778, - "fn": 22, - "accuracy": 0.9725 - } - }, - "auroc": 0.9884622395833333 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.94539375 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.94539375 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9435343749999999 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9435343749999999 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 333, - "fn": 67, - "accuracy": 0.8325 - } - }, - "auroc": 0.9444640625 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 333, - "fn": 67, - "accuracy": 0.8325 - } - }, - "auroc": 0.9444640625 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - }, - "0.01": { - "tp": 128, - "fn": 72, - "accuracy": 0.64 - } - }, - "auroc": 0.8865614583333334 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - }, - "0.01": { - "tp": 128, - "fn": 72, - "accuracy": 0.64 - } - }, - "auroc": 0.8865614583333334 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 149, - "fn": 51, - "accuracy": 0.745 - }, - "0.01": { - "tp": 118, - "fn": 82, - "accuracy": 0.59 - } - }, - "auroc": 0.859703125 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 149, - "fn": 51, - "accuracy": 0.745 - }, - "0.01": { - "tp": 118, - "fn": 82, - "accuracy": 0.59 - } - }, - "auroc": 0.859703125 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 314, - "fn": 86, - "accuracy": 0.785 - }, - "0.01": { - "tp": 246, - "fn": 154, - "accuracy": 0.615 - } - }, - "auroc": 0.8731322916666666 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 314, - "fn": 86, - "accuracy": 0.785 - }, - "0.01": { - "tp": 246, - "fn": 154, - "accuracy": 0.615 - } - }, - "auroc": 0.8731322916666666 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.99743125 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.99743125 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9918510416666666 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9918510416666666 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9946411458333333 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9946411458333333 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.995784375 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.995784375 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.968521875 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.968521875 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.982153125 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.982153125 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 160, - "fn": 40, - "accuracy": 0.8 - }, - "0.01": { - "tp": 136, - "fn": 64, - "accuracy": 0.68 - } - }, - "auroc": 0.8909145833333333 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 160, - "fn": 40, - "accuracy": 0.8 - }, - "0.01": { - "tp": 136, - "fn": 64, - "accuracy": 0.68 - } - }, - "auroc": 0.8909145833333333 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 140, - "fn": 60, - "accuracy": 0.7 - }, - "0.01": { - "tp": 114, - "fn": 86, - "accuracy": 0.57 - } - }, - "auroc": 0.856365625 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 140, - "fn": 60, - "accuracy": 0.7 - }, - "0.01": { - "tp": 114, - "fn": 86, - "accuracy": 0.57 - } - }, - "auroc": 0.856365625 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 300, - "fn": 100, - "accuracy": 0.75 - }, - "0.01": { - "tp": 250, - "fn": 150, - "accuracy": 0.625 - } - }, - "auroc": 0.8736401041666666 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 300, - "fn": 100, - "accuracy": 0.75 - }, - "0.01": { - "tp": 250, - "fn": 150, - "accuracy": 0.625 - } - }, - "auroc": 0.8736401041666666 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2086, - "fn": 114, - "accuracy": 0.9481818181818182 - }, - "0.01": { - "tp": 1974, - "fn": 226, - "accuracy": 0.8972727272727272 - } - }, - "auroc": 0.9618183712121212 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1190, - "fn": 10, - "accuracy": 0.9916666666666667 - }, - "0.01": { - "tp": 1180, - "fn": 20, - "accuracy": 0.9833333333333333 - } - }, - "auroc": 0.9932630208333334 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3276, - "fn": 124, - "accuracy": 0.9635294117647059 - }, - "0.01": { - "tp": 3154, - "fn": 246, - "accuracy": 0.9276470588235294 - } - }, - "auroc": 0.9729164828431371 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1984, - "fn": 216, - "accuracy": 0.9018181818181819 - }, - "0.01": { - "tp": 1795, - "fn": 405, - "accuracy": 0.8159090909090909 - } - }, - "auroc": 0.9407855113636365 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1169, - "fn": 31, - "accuracy": 0.9741666666666666 - }, - "0.01": { - "tp": 1135, - "fn": 65, - "accuracy": 0.9458333333333333 - } - }, - "auroc": 0.9773628472222222 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3153, - "fn": 247, - "accuracy": 0.9273529411764706 - }, - "0.01": { - "tp": 2930, - "fn": 470, - "accuracy": 0.861764705882353 - } - }, - "auroc": 0.9536951593137254 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 4070, - "fn": 330, - "accuracy": 0.925 - }, - "0.01": { - "tp": 3769, - "fn": 631, - "accuracy": 0.8565909090909091 - } - }, - "auroc": 0.9513019412878788 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2359, - "fn": 41, - "accuracy": 0.9829166666666667 - }, - "0.01": { - "tp": 2315, - "fn": 85, - "accuracy": 0.9645833333333333 - } - }, - "auroc": 0.9853129340277778 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 6429, - "fn": 371, - "accuracy": 0.9454411764705882 - }, - "0.01": { - "tp": 6084, - "fn": 716, - "accuracy": 0.8947058823529411 - } - }, - "auroc": 0.9633058210784314 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999940625 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994770833333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997088541666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998208333333333 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.99056875 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9951947916666668 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998807291666667 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9950229166666666 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - } - }, - "auroc": 0.9974518229166667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9655302083333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9894302083333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9774802083333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 146, - "fn": 54, - "accuracy": 0.73 - } - }, - "auroc": 0.9204427083333333 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9737885416666667 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 336, - "fn": 64, - "accuracy": 0.84 - } - }, - "auroc": 0.947115625 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9429864583333334 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.981609375 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 717, - "fn": 83, - "accuracy": 0.89625 - } - }, - "auroc": 0.9622979166666668 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9792197916666667 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9814343750000001 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9803270833333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9701333333333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9613333333333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - } - }, - "auroc": 0.9657333333333333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9746765625 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9713838541666666 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 736, - "fn": 64, - "accuracy": 0.92 - } - }, - "auroc": 0.9730302083333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9402791666666667 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981552083333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9692171875 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 124, - "fn": 76, - "accuracy": 0.62 - } - }, - "auroc": 0.9045125 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9606302083333333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 310, - "fn": 90, - "accuracy": 0.775 - } - }, - "auroc": 0.9325713541666666 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 301, - "fn": 99, - "accuracy": 0.7525 - } - }, - "auroc": 0.9223958333333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9793927083333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 771, - "fn": 29, - "accuracy": 0.96375 - }, - "0.01": { - "tp": 687, - "fn": 113, - "accuracy": 0.85875 - } - }, - "auroc": 0.9508942708333333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.989121875 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9921854166666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9906536458333335 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 154, - "fn": 46, - "accuracy": 0.77 - }, - "0.01": { - "tp": 109, - "fn": 91, - "accuracy": 0.545 - } - }, - "auroc": 0.8693427083333332 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9712729166666666 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 348, - "fn": 52, - "accuracy": 0.87 - }, - "0.01": { - "tp": 297, - "fn": 103, - "accuracy": 0.7425 - } - }, - "auroc": 0.9203078124999999 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - }, - "0.01": { - "tp": 306, - "fn": 94, - "accuracy": 0.765 - } - }, - "auroc": 0.9292322916666667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9817291666666668 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 747, - "fn": 53, - "accuracy": 0.93375 - }, - "0.01": { - "tp": 692, - "fn": 108, - "accuracy": 0.865 - } - }, - "auroc": 0.9554807291666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.99126875 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9937770833333335 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9925229166666667 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.987225 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9757291666666666 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9814770833333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9892468750000001 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.984753125 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 786, - "fn": 14, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 774, - "fn": 26, - "accuracy": 0.9675 - } - }, - "auroc": 0.987 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9432739583333333 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9432739583333333 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9426343749999999 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9426343749999999 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9429541666666668 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9429541666666668 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 109, - "fn": 91, - "accuracy": 0.545 - } - }, - "auroc": 0.8594041666666667 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 109, - "fn": 91, - "accuracy": 0.545 - } - }, - "auroc": 0.8594041666666667 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - }, - "0.01": { - "tp": 98, - "fn": 102, - "accuracy": 0.49 - } - }, - "auroc": 0.8146375 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 134, - "fn": 66, - "accuracy": 0.67 - }, - "0.01": { - "tp": 98, - "fn": 102, - "accuracy": 0.49 - } - }, - "auroc": 0.8146375 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 285, - "fn": 115, - "accuracy": 0.7125 - }, - "0.01": { - "tp": 207, - "fn": 193, - "accuracy": 0.5175 - } - }, - "auroc": 0.8370208333333334 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 285, - "fn": 115, - "accuracy": 0.7125 - }, - "0.01": { - "tp": 207, - "fn": 193, - "accuracy": 0.5175 - } - }, - "auroc": 0.8370208333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9977083333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9977083333333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9921552083333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9921552083333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9949317708333334 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9949317708333334 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9927166666666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9927166666666667 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9543520833333333 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9543520833333333 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9735343750000001 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9735343750000001 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 125, - "fn": 75, - "accuracy": 0.625 - } - }, - "auroc": 0.8723614583333332 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - }, - "0.01": { - "tp": 125, - "fn": 75, - "accuracy": 0.625 - } - }, - "auroc": 0.8723614583333332 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - }, - "0.01": { - "tp": 104, - "fn": 96, - "accuracy": 0.52 - } - }, - "auroc": 0.8267635416666665 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - }, - "0.01": { - "tp": 104, - "fn": 96, - "accuracy": 0.52 - } - }, - "auroc": 0.8267635416666665 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 281, - "fn": 119, - "accuracy": 0.7025 - }, - "0.01": { - "tp": 229, - "fn": 171, - "accuracy": 0.5725 - } - }, - "auroc": 0.8495625 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 281, - "fn": 119, - "accuracy": 0.7025 - }, - "0.01": { - "tp": 229, - "fn": 171, - "accuracy": 0.5725 - } - }, - "auroc": 0.8495625 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2063, - "fn": 137, - "accuracy": 0.9377272727272727 - }, - "0.01": { - "tp": 1940, - "fn": 260, - "accuracy": 0.8818181818181818 - } - }, - "auroc": 0.9573477272727273 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1189, - "fn": 11, - "accuracy": 0.9908333333333333 - }, - "0.01": { - "tp": 1179, - "fn": 21, - "accuracy": 0.9825 - } - }, - "auroc": 0.9924098958333334 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3252, - "fn": 148, - "accuracy": 0.9564705882352941 - }, - "0.01": { - "tp": 3119, - "fn": 281, - "accuracy": 0.9173529411764706 - } - }, - "auroc": 0.9697226102941177 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1932, - "fn": 268, - "accuracy": 0.8781818181818182 - }, - "0.01": { - "tp": 1688, - "fn": 512, - "accuracy": 0.7672727272727272 - } - }, - "auroc": 0.9256381628787878 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1167, - "fn": 33, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 1124, - "fn": 76, - "accuracy": 0.9366666666666666 - } - }, - "auroc": 0.9722204861111112 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3099, - "fn": 301, - "accuracy": 0.9114705882352941 - }, - "0.01": { - "tp": 2812, - "fn": 588, - "accuracy": 0.8270588235294117 - } - }, - "auroc": 0.9420789828431373 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3995, - "fn": 405, - "accuracy": 0.9079545454545455 - }, - "0.01": { - "tp": 3628, - "fn": 772, - "accuracy": 0.8245454545454546 - } - }, - "auroc": 0.9414929450757574 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2356, - "fn": 44, - "accuracy": 0.9816666666666667 - }, - "0.01": { - "tp": 2303, - "fn": 97, - "accuracy": 0.9595833333333333 - } - }, - "auroc": 0.9823151909722222 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 6351, - "fn": 449, - "accuracy": 0.9339705882352941 - }, - "0.01": { - "tp": 5931, - "fn": 869, - "accuracy": 0.8722058823529412 - } - }, - "auroc": 0.9559007965686273 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.997077517361111 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2397, - "fn": 3, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 2383, - "fn": 17, - "accuracy": 0.9929166666666667 - } - }, - "auroc": 0.9944907118055555 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4797, - "fn": 3, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 4783, - "fn": 17, - "accuracy": 0.9964583333333333 - } - }, - "auroc": 0.9957841145833334 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2397, - "fn": 3, - "accuracy": 0.99875 - } - }, - "auroc": 0.9965383680555555 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2388, - "fn": 12, - "accuracy": 0.995 - }, - "0.01": { - "tp": 2328, - "fn": 72, - "accuracy": 0.97 - } - }, - "auroc": 0.9867977430555556 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4788, - "fn": 12, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 4725, - "fn": 75, - "accuracy": 0.984375 - } - }, - "auroc": 0.9916680555555556 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4797, - "fn": 3, - "accuracy": 0.999375 - } - }, - "auroc": 0.9968079427083334 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4785, - "fn": 15, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 4711, - "fn": 89, - "accuracy": 0.9814583333333333 - } - }, - "auroc": 0.9906442274305556 - }, - { - "domain": "reddit", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9585, - "fn": 15, - "accuracy": 0.9984375 - }, - "0.01": { - "tp": 9508, - "fn": 92, - "accuracy": 0.9904166666666666 - } - }, - "auroc": 0.9937260850694445 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2385, - "fn": 15, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 2226, - "fn": 174, - "accuracy": 0.9275 - } - }, - "auroc": 0.96612421875 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2343, - "fn": 57, - "accuracy": 0.97625 - }, - "0.01": { - "tp": 2282, - "fn": 118, - "accuracy": 0.9508333333333333 - } - }, - "auroc": 0.9828407986111112 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4728, - "fn": 72, - "accuracy": 0.985 - }, - "0.01": { - "tp": 4508, - "fn": 292, - "accuracy": 0.9391666666666667 - } - }, - "auroc": 0.9744825086805555 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2241, - "fn": 159, - "accuracy": 0.93375 - }, - "0.01": { - "tp": 1927, - "fn": 473, - "accuracy": 0.8029166666666666 - } - }, - "auroc": 0.9380826388888888 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2254, - "fn": 146, - "accuracy": 0.9391666666666667 - }, - "0.01": { - "tp": 2161, - "fn": 239, - "accuracy": 0.9004166666666666 - } - }, - "auroc": 0.9614404513888889 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4495, - "fn": 305, - "accuracy": 0.9364583333333333 - }, - "0.01": { - "tp": 4088, - "fn": 712, - "accuracy": 0.8516666666666667 - } - }, - "auroc": 0.9497615451388889 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4626, - "fn": 174, - "accuracy": 0.96375 - }, - "0.01": { - "tp": 4153, - "fn": 647, - "accuracy": 0.8652083333333334 - } - }, - "auroc": 0.9521034288194445 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4597, - "fn": 203, - "accuracy": 0.9577083333333334 - }, - "0.01": { - "tp": 4443, - "fn": 357, - "accuracy": 0.925625 - } - }, - "auroc": 0.972140625 - }, - { - "domain": "reddit", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9223, - "fn": 377, - "accuracy": 0.9607291666666666 - }, - "0.01": { - "tp": 8596, - "fn": 1004, - "accuracy": 0.8954166666666666 - } - }, - "auroc": 0.9621220269097222 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2306, - "fn": 94, - "accuracy": 0.9608333333333333 - }, - "0.01": { - "tp": 2189, - "fn": 211, - "accuracy": 0.9120833333333334 - } - }, - "auroc": 0.9722682291666667 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2319, - "fn": 81, - "accuracy": 0.96625 - }, - "0.01": { - "tp": 2211, - "fn": 189, - "accuracy": 0.92125 - } - }, - "auroc": 0.9737786458333334 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4625, - "fn": 175, - "accuracy": 0.9635416666666666 - }, - "0.01": { - "tp": 4400, - "fn": 400, - "accuracy": 0.9166666666666666 - } - }, - "auroc": 0.9730234375000001 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2271, - "fn": 129, - "accuracy": 0.94625 - }, - "0.01": { - "tp": 2134, - "fn": 266, - "accuracy": 0.8891666666666667 - } - }, - "auroc": 0.9628106770833333 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2269, - "fn": 131, - "accuracy": 0.9454166666666667 - }, - "0.01": { - "tp": 2087, - "fn": 313, - "accuracy": 0.8695833333333334 - } - }, - "auroc": 0.9562391493055555 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4540, - "fn": 260, - "accuracy": 0.9458333333333333 - }, - "0.01": { - "tp": 4221, - "fn": 579, - "accuracy": 0.879375 - } - }, - "auroc": 0.9595249131944444 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4577, - "fn": 223, - "accuracy": 0.9535416666666666 - }, - "0.01": { - "tp": 4323, - "fn": 477, - "accuracy": 0.900625 - } - }, - "auroc": 0.9675394531249999 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4588, - "fn": 212, - "accuracy": 0.9558333333333333 - }, - "0.01": { - "tp": 4298, - "fn": 502, - "accuracy": 0.8954166666666666 - } - }, - "auroc": 0.9650088975694444 - }, - { - "domain": "reddit", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9165, - "fn": 435, - "accuracy": 0.9546875 - }, - "0.01": { - "tp": 8621, - "fn": 979, - "accuracy": 0.8980208333333334 - } - }, - "auroc": 0.9662741753472222 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2360, - "fn": 40, - "accuracy": 0.9833333333333333 - }, - "0.01": { - "tp": 2082, - "fn": 318, - "accuracy": 0.8675 - } - }, - "auroc": 0.9413356770833334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2397, - "fn": 3, - "accuracy": 0.99875 - } - }, - "auroc": 0.9961026041666666 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4760, - "fn": 40, - "accuracy": 0.9916666666666667 - }, - "0.01": { - "tp": 4479, - "fn": 321, - "accuracy": 0.933125 - } - }, - "auroc": 0.9687191406250001 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2215, - "fn": 185, - "accuracy": 0.9229166666666667 - }, - "0.01": { - "tp": 1754, - "fn": 646, - "accuracy": 0.7308333333333333 - } - }, - "auroc": 0.9250130208333334 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2331, - "fn": 69, - "accuracy": 0.97125 - }, - "0.01": { - "tp": 2193, - "fn": 207, - "accuracy": 0.91375 - } - }, - "auroc": 0.9625157986111111 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4546, - "fn": 254, - "accuracy": 0.9470833333333334 - }, - "0.01": { - "tp": 3947, - "fn": 853, - "accuracy": 0.8222916666666666 - } - }, - "auroc": 0.9437644097222222 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4575, - "fn": 225, - "accuracy": 0.953125 - }, - "0.01": { - "tp": 3836, - "fn": 964, - "accuracy": 0.7991666666666667 - } - }, - "auroc": 0.9331743489583333 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4731, - "fn": 69, - "accuracy": 0.985625 - }, - "0.01": { - "tp": 4590, - "fn": 210, - "accuracy": 0.95625 - } - }, - "auroc": 0.9793092013888889 - }, - { - "domain": "reddit", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9306, - "fn": 294, - "accuracy": 0.969375 - }, - "0.01": { - "tp": 8426, - "fn": 1174, - "accuracy": 0.8777083333333333 - } - }, - "auroc": 0.9562417751736112 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 2349, - "fn": 51, - "accuracy": 0.97875 - } - }, - "auroc": 0.9875296874999999 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2383, - "fn": 17, - "accuracy": 0.9929166666666667 - }, - "0.01": { - "tp": 2346, - "fn": 54, - "accuracy": 0.9775 - } - }, - "auroc": 0.9881568576388888 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4781, - "fn": 19, - "accuracy": 0.9960416666666667 - }, - "0.01": { - "tp": 4695, - "fn": 105, - "accuracy": 0.978125 - } - }, - "auroc": 0.9878432725694445 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2004, - "fn": 396, - "accuracy": 0.835 - }, - "0.01": { - "tp": 1562, - "fn": 838, - "accuracy": 0.6508333333333334 - } - }, - "auroc": 0.8965466145833333 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2299, - "fn": 101, - "accuracy": 0.9579166666666666 - }, - "0.01": { - "tp": 2161, - "fn": 239, - "accuracy": 0.9004166666666666 - } - }, - "auroc": 0.9630349826388889 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4303, - "fn": 497, - "accuracy": 0.8964583333333334 - }, - "0.01": { - "tp": 3723, - "fn": 1077, - "accuracy": 0.775625 - } - }, - "auroc": 0.929790798611111 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4402, - "fn": 398, - "accuracy": 0.9170833333333334 - }, - "0.01": { - "tp": 3911, - "fn": 889, - "accuracy": 0.8147916666666667 - } - }, - "auroc": 0.9420381510416667 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4682, - "fn": 118, - "accuracy": 0.9754166666666667 - }, - "0.01": { - "tp": 4507, - "fn": 293, - "accuracy": 0.9389583333333333 - } - }, - "auroc": 0.9755959201388889 - }, - { - "domain": "reddit", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9084, - "fn": 516, - "accuracy": 0.94625 - }, - "0.01": { - "tp": 8418, - "fn": 1182, - "accuracy": 0.876875 - } - }, - "auroc": 0.9588170355902778 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2359, - "fn": 41, - "accuracy": 0.9829166666666667 - }, - "0.01": { - "tp": 2342, - "fn": 58, - "accuracy": 0.9758333333333333 - } - }, - "auroc": 0.9875138888888888 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2372, - "fn": 28, - "accuracy": 0.9883333333333333 - }, - "0.01": { - "tp": 2341, - "fn": 59, - "accuracy": 0.9754166666666667 - } - }, - "auroc": 0.9892695312500001 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4731, - "fn": 69, - "accuracy": 0.985625 - }, - "0.01": { - "tp": 4683, - "fn": 117, - "accuracy": 0.975625 - } - }, - "auroc": 0.9883917100694445 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2342, - "fn": 58, - "accuracy": 0.9758333333333333 - }, - "0.01": { - "tp": 2272, - "fn": 128, - "accuracy": 0.9466666666666667 - } - }, - "auroc": 0.9819737847222223 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2336, - "fn": 64, - "accuracy": 0.9733333333333334 - }, - "0.01": { - "tp": 2234, - "fn": 166, - "accuracy": 0.9308333333333333 - } - }, - "auroc": 0.9727223958333333 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4678, - "fn": 122, - "accuracy": 0.9745833333333334 - }, - "0.01": { - "tp": 4506, - "fn": 294, - "accuracy": 0.93875 - } - }, - "auroc": 0.9773480902777778 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4701, - "fn": 99, - "accuracy": 0.979375 - }, - "0.01": { - "tp": 4614, - "fn": 186, - "accuracy": 0.96125 - } - }, - "auroc": 0.9847438368055557 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4708, - "fn": 92, - "accuracy": 0.9808333333333333 - }, - "0.01": { - "tp": 4575, - "fn": 225, - "accuracy": 0.953125 - } - }, - "auroc": 0.9809959635416666 - }, - { - "domain": "reddit", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9409, - "fn": 191, - "accuracy": 0.9801041666666667 - }, - "0.01": { - "tp": 9189, - "fn": 411, - "accuracy": 0.9571875 - } - }, - "auroc": 0.9828699001736111 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2138, - "fn": 262, - "accuracy": 0.8908333333333334 - }, - "0.01": { - "tp": 1975, - "fn": 425, - "accuracy": 0.8229166666666666 - } - }, - "auroc": 0.9387500000000001 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2138, - "fn": 262, - "accuracy": 0.8908333333333334 - }, - "0.01": { - "tp": 1975, - "fn": 425, - "accuracy": 0.8229166666666666 - } - }, - "auroc": 0.9387500000000001 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2121, - "fn": 279, - "accuracy": 0.88375 - }, - "0.01": { - "tp": 1945, - "fn": 455, - "accuracy": 0.8104166666666667 - } - }, - "auroc": 0.9382809895833335 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2121, - "fn": 279, - "accuracy": 0.88375 - }, - "0.01": { - "tp": 1945, - "fn": 455, - "accuracy": 0.8104166666666667 - } - }, - "auroc": 0.9382809895833335 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4259, - "fn": 541, - "accuracy": 0.8872916666666667 - }, - "0.01": { - "tp": 3920, - "fn": 880, - "accuracy": 0.8166666666666667 - } - }, - "auroc": 0.9385154947916665 - }, - { - "domain": "reddit", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4259, - "fn": 541, - "accuracy": 0.8872916666666667 - }, - "0.01": { - "tp": 3920, - "fn": 880, - "accuracy": 0.8166666666666667 - } - }, - "auroc": 0.9385154947916665 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 1901, - "fn": 499, - "accuracy": 0.7920833333333334 - }, - "0.01": { - "tp": 1480, - "fn": 920, - "accuracy": 0.6166666666666667 - } - }, - "auroc": 0.8777271701388889 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 1901, - "fn": 499, - "accuracy": 0.7920833333333334 - }, - "0.01": { - "tp": 1480, - "fn": 920, - "accuracy": 0.6166666666666667 - } - }, - "auroc": 0.8777271701388889 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 1774, - "fn": 626, - "accuracy": 0.7391666666666666 - }, - "0.01": { - "tp": 1347, - "fn": 1053, - "accuracy": 0.56125 - } - }, - "auroc": 0.8464271701388888 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 1774, - "fn": 626, - "accuracy": 0.7391666666666666 - }, - "0.01": { - "tp": 1347, - "fn": 1053, - "accuracy": 0.56125 - } - }, - "auroc": 0.8464271701388888 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 3675, - "fn": 1125, - "accuracy": 0.765625 - }, - "0.01": { - "tp": 2827, - "fn": 1973, - "accuracy": 0.5889583333333334 - } - }, - "auroc": 0.862077170138889 - }, - { - "domain": "reddit", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 3675, - "fn": 1125, - "accuracy": 0.765625 - }, - "0.01": { - "tp": 2827, - "fn": 1973, - "accuracy": 0.5889583333333334 - } - }, - "auroc": 0.862077170138889 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2385, - "fn": 15, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 2370, - "fn": 30, - "accuracy": 0.9875 - } - }, - "auroc": 0.99186875 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2385, - "fn": 15, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 2370, - "fn": 30, - "accuracy": 0.9875 - } - }, - "auroc": 0.99186875 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2370, - "fn": 30, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 2336, - "fn": 64, - "accuracy": 0.9733333333333334 - } - }, - "auroc": 0.9862209201388888 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2370, - "fn": 30, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 2336, - "fn": 64, - "accuracy": 0.9733333333333334 - } - }, - "auroc": 0.9862209201388888 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4755, - "fn": 45, - "accuracy": 0.990625 - }, - "0.01": { - "tp": 4706, - "fn": 94, - "accuracy": 0.9804166666666667 - } - }, - "auroc": 0.9890448350694445 - }, - { - "domain": "reddit", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4755, - "fn": 45, - "accuracy": 0.990625 - }, - "0.01": { - "tp": 4706, - "fn": 94, - "accuracy": 0.9804166666666667 - } - }, - "auroc": 0.9890448350694445 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2369, - "fn": 31, - "accuracy": 0.9870833333333333 - }, - "0.01": { - "tp": 2343, - "fn": 57, - "accuracy": 0.97625 - } - }, - "auroc": 0.9889625 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2369, - "fn": 31, - "accuracy": 0.9870833333333333 - }, - "0.01": { - "tp": 2343, - "fn": 57, - "accuracy": 0.97625 - } - }, - "auroc": 0.9889625 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2313, - "fn": 87, - "accuracy": 0.96375 - }, - "0.01": { - "tp": 2119, - "fn": 281, - "accuracy": 0.8829166666666667 - } - }, - "auroc": 0.957275 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2313, - "fn": 87, - "accuracy": 0.96375 - }, - "0.01": { - "tp": 2119, - "fn": 281, - "accuracy": 0.8829166666666667 - } - }, - "auroc": 0.957275 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4682, - "fn": 118, - "accuracy": 0.9754166666666667 - }, - "0.01": { - "tp": 4462, - "fn": 338, - "accuracy": 0.9295833333333333 - } - }, - "auroc": 0.97311875 - }, - { - "domain": "reddit", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4682, - "fn": 118, - "accuracy": 0.9754166666666667 - }, - "0.01": { - "tp": 4462, - "fn": 338, - "accuracy": 0.9295833333333333 - } - }, - "auroc": 0.97311875 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 1901, - "fn": 499, - "accuracy": 0.7920833333333334 - }, - "0.01": { - "tp": 1587, - "fn": 813, - "accuracy": 0.66125 - } - }, - "auroc": 0.8859059895833333 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 1901, - "fn": 499, - "accuracy": 0.7920833333333334 - }, - "0.01": { - "tp": 1587, - "fn": 813, - "accuracy": 0.66125 - } - }, - "auroc": 0.8859059895833333 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 1676, - "fn": 724, - "accuracy": 0.6983333333333334 - }, - "0.01": { - "tp": 1340, - "fn": 1060, - "accuracy": 0.5583333333333333 - } - }, - "auroc": 0.8473663194444444 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 1676, - "fn": 724, - "accuracy": 0.6983333333333334 - }, - "0.01": { - "tp": 1340, - "fn": 1060, - "accuracy": 0.5583333333333333 - } - }, - "auroc": 0.8473663194444444 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 3577, - "fn": 1223, - "accuracy": 0.7452083333333334 - }, - "0.01": { - "tp": 2927, - "fn": 1873, - "accuracy": 0.6097916666666666 - } - }, - "auroc": 0.8666361545138889 - }, - { - "domain": "reddit", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 3577, - "fn": 1223, - "accuracy": 0.7452083333333334 - }, - "0.01": { - "tp": 2927, - "fn": 1873, - "accuracy": 0.6097916666666666 - } - }, - "auroc": 0.8666361545138889 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 24902, - "fn": 1498, - "accuracy": 0.9432575757575757 - }, - "0.01": { - "tp": 23343, - "fn": 3057, - "accuracy": 0.8842045454545454 - } - }, - "auroc": 0.9577330571338384 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 14214, - "fn": 186, - "accuracy": 0.9870833333333333 - }, - "0.01": { - "tp": 13960, - "fn": 440, - "accuracy": 0.9694444444444444 - } - }, - "auroc": 0.9874398582175926 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 39116, - "fn": 1684, - "accuracy": 0.9587254901960784 - }, - "0.01": { - "tp": 37303, - "fn": 3497, - "accuracy": 0.9142892156862745 - } - }, - "auroc": 0.9682178104575163 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 23727, - "fn": 2673, - "accuracy": 0.89875 - }, - "0.01": { - "tp": 21133, - "fn": 5267, - "accuracy": 0.8004924242424243 - } - }, - "auroc": 0.9342305003156566 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 13877, - "fn": 523, - "accuracy": 0.9636805555555555 - }, - "0.01": { - "tp": 13164, - "fn": 1236, - "accuracy": 0.9141666666666667 - } - }, - "auroc": 0.9671250868055554 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 37604, - "fn": 3196, - "accuracy": 0.9216666666666666 - }, - "0.01": { - "tp": 34297, - "fn": 6503, - "accuracy": 0.8406127450980392 - } - }, - "auroc": 0.9458403543709151 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 48629, - "fn": 4171, - "accuracy": 0.9210037878787879 - }, - "0.01": { - "tp": 44476, - "fn": 8324, - "accuracy": 0.8423484848484849 - } - }, - "auroc": 0.9459817787247473 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 28091, - "fn": 709, - "accuracy": 0.9753819444444445 - }, - "0.01": { - "tp": 27124, - "fn": 1676, - "accuracy": 0.9418055555555556 - } - }, - "auroc": 0.9772824725115741 - }, - { - "domain": "reddit", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 76720, - "fn": 4880, - "accuracy": 0.9401960784313725 - }, - "0.01": { - "tp": 71600, - "fn": 10000, - "accuracy": 0.8774509803921569 - } - }, - "auroc": 0.9570290824142157 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998010416666666 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996979166666666 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997494791666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99979375 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9954125 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9976031249999999 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997973958333334 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9975552083333334 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9986763020833334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.98680625 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9942083333333332 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9905072916666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9720437499999999 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9724552083333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9722494791666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.979425 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9833317708333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - } - }, - "auroc": 0.9813783854166667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992260416666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.993 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9961130208333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980937500000001 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9776166666666666 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9878552083333334 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986598958333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9853083333333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - } - }, - "auroc": 0.9919841145833334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9554854166666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9971010416666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - } - }, - "auroc": 0.9762932291666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9691489583333333 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.965540625 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9673447916666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9623171875000001 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9813208333333334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 738, - "fn": 62, - "accuracy": 0.9225 - } - }, - "auroc": 0.9718190104166666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9854802083333334 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9939656250000001 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9897229166666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9469208333333333 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9784104166666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9626656250000001 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - } - }, - "auroc": 0.9662005208333333 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9861880208333333 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 756, - "fn": 44, - "accuracy": 0.945 - } - }, - "auroc": 0.9761942708333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996781250000001 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998552083333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997666666666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996135416666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9980364583333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9988250000000001 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996458333333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9989458333333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9992958333333334 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.990340625 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.990340625 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9841791666666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9841791666666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9872598958333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9872598958333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9698708333333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9698708333333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.955978125 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.955978125 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9629244791666666 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9629244791666666 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999635416666667 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999635416666667 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999645833333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999645833333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999640625 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999640625 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999072916666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999072916666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9962395833333333 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9962395833333333 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980734375 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980734375 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9891885416666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9891885416666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9669270833333334 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9669270833333334 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9780578125 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9780578125 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2194, - "fn": 6, - "accuracy": 0.9972727272727273 - }, - "0.01": { - "tp": 2134, - "fn": 66, - "accuracy": 0.97 - } - }, - "auroc": 0.9887043560606061 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1196, - "fn": 4, - "accuracy": 0.9966666666666667 - } - }, - "auroc": 0.9963046875 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3392, - "fn": 8, - "accuracy": 0.9976470588235294 - }, - "0.01": { - "tp": 3330, - "fn": 70, - "accuracy": 0.9794117647058823 - } - }, - "auroc": 0.9913868259803922 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2164, - "fn": 36, - "accuracy": 0.9836363636363636 - }, - "0.01": { - "tp": 2085, - "fn": 115, - "accuracy": 0.9477272727272728 - } - }, - "auroc": 0.980809375 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1184, - "fn": 16, - "accuracy": 0.9866666666666667 - }, - "0.01": { - "tp": 1156, - "fn": 44, - "accuracy": 0.9633333333333334 - } - }, - "auroc": 0.9812453125 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3348, - "fn": 52, - "accuracy": 0.9847058823529412 - }, - "0.01": { - "tp": 3241, - "fn": 159, - "accuracy": 0.9532352941176471 - } - }, - "auroc": 0.9809632352941178 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 4358, - "fn": 42, - "accuracy": 0.9904545454545455 - }, - "0.01": { - "tp": 4219, - "fn": 181, - "accuracy": 0.9588636363636364 - } - }, - "auroc": 0.9847568655303031 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2382, - "fn": 18, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 2352, - "fn": 48, - "accuracy": 0.98 - } - }, - "auroc": 0.988775 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 6740, - "fn": 60, - "accuracy": 0.9911764705882353 - }, - "0.01": { - "tp": 6571, - "fn": 229, - "accuracy": 0.9663235294117647 - } - }, - "auroc": 0.9861750306372549 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998010416666666 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996979166666666 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997494791666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99979375 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9954125 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9976031249999999 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997973958333334 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9975552083333334 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9986763020833334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.98680625 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9942083333333332 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9905072916666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9720437499999999 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9724552083333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9722494791666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.979425 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9833317708333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - } - }, - "auroc": 0.9813783854166667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992260416666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.993 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9961130208333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980937500000001 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9776166666666666 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9878552083333334 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986598958333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9853083333333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - } - }, - "auroc": 0.9919841145833334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9554854166666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9971010416666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - } - }, - "auroc": 0.9762932291666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9691489583333333 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.965540625 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9673447916666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9623171875000001 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9813208333333334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 738, - "fn": 62, - "accuracy": 0.9225 - } - }, - "auroc": 0.9718190104166666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9854802083333334 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9939656250000001 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9897229166666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9469208333333333 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9784104166666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9626656250000001 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - } - }, - "auroc": 0.9662005208333333 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9861880208333333 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 756, - "fn": 44, - "accuracy": 0.945 - } - }, - "auroc": 0.9761942708333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996781250000001 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998552083333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997666666666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996135416666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9980364583333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9988250000000001 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996458333333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9989458333333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9992958333333334 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.990340625 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.990340625 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9841791666666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9841791666666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9872598958333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9872598958333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9698708333333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9698708333333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.955978125 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.955978125 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9629244791666666 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9629244791666666 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999635416666667 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999635416666667 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999645833333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999645833333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999640625 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999640625 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999072916666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999072916666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9962395833333333 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9962395833333333 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980734375 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980734375 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9891885416666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9891885416666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9669270833333334 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9669270833333334 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9780578125 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9780578125 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2194, - "fn": 6, - "accuracy": 0.9972727272727273 - }, - "0.01": { - "tp": 2134, - "fn": 66, - "accuracy": 0.97 - } - }, - "auroc": 0.9887043560606061 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1196, - "fn": 4, - "accuracy": 0.9966666666666667 - } - }, - "auroc": 0.9963046875 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3392, - "fn": 8, - "accuracy": 0.9976470588235294 - }, - "0.01": { - "tp": 3330, - "fn": 70, - "accuracy": 0.9794117647058823 - } - }, - "auroc": 0.9913868259803922 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2164, - "fn": 36, - "accuracy": 0.9836363636363636 - }, - "0.01": { - "tp": 2085, - "fn": 115, - "accuracy": 0.9477272727272728 - } - }, - "auroc": 0.980809375 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1184, - "fn": 16, - "accuracy": 0.9866666666666667 - }, - "0.01": { - "tp": 1156, - "fn": 44, - "accuracy": 0.9633333333333334 - } - }, - "auroc": 0.9812453125 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3348, - "fn": 52, - "accuracy": 0.9847058823529412 - }, - "0.01": { - "tp": 3241, - "fn": 159, - "accuracy": 0.9532352941176471 - } - }, - "auroc": 0.9809632352941178 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 4358, - "fn": 42, - "accuracy": 0.9904545454545455 - }, - "0.01": { - "tp": 4219, - "fn": 181, - "accuracy": 0.9588636363636364 - } - }, - "auroc": 0.9847568655303031 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2382, - "fn": 18, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 2352, - "fn": 48, - "accuracy": 0.98 - } - }, - "auroc": 0.988775 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 6740, - "fn": 60, - "accuracy": 0.9911764705882353 - }, - "0.01": { - "tp": 6571, - "fn": 229, - "accuracy": 0.9663235294117647 - } - }, - "auroc": 0.9861750306372549 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985854166666668 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.997009375 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9977973958333334 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9974135416666666 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99275625 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9950848958333334 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9979994791666665 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9948828125000001 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9964411458333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9969645833333334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9929677083333334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9949661458333334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9909708333333332 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9698729166666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.980421875 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9939677083333334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9814203125000001 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 781, - "fn": 19, - "accuracy": 0.97625 - } - }, - "auroc": 0.9876940104166667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998065625 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9950041666666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9965348958333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9968947916666666 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9830916666666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9899932291666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9974802083333334 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9890479166666666 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9932640624999999 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9916531249999999 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990145833333333 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9953338541666668 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.98750625 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9815041666666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9845052083333333 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9895796875 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9902593749999999 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 784, - "fn": 16, - "accuracy": 0.98 - } - }, - "auroc": 0.98991953125 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99676875 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9973104166666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9970395833333334 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9820260416666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9861968749999999 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9841114583333334 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9893973958333334 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9917536458333334 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9905755208333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993218749999999 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983927083333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9988572916666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9968989583333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9931354166666666 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9950171875 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981104166666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9957640624999999 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9969372395833334 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9925364583333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9925364583333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9856145833333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9856145833333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9890755208333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9890755208333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9830916666666667 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9830916666666667 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9727875 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9727875 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9779395833333334 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9779395833333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9977843750000001 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9977843750000001 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9940010416666667 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9940010416666667 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9958927083333333 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9958927083333333 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9970927083333334 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9970927083333334 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9840760416666666 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9840760416666666 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.990584375 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.990584375 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9925354166666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9925354166666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9777260416666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9777260416666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9851307291666668 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9851307291666668 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2199, - "fn": 1, - "accuracy": 0.9995454545454545 - }, - "0.01": { - "tp": 2192, - "fn": 8, - "accuracy": 0.9963636363636363 - } - }, - "auroc": 0.9949454545454546 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9966164930555554 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3397, - "fn": 3, - "accuracy": 0.9991176470588236 - }, - "0.01": { - "tp": 3390, - "fn": 10, - "accuracy": 0.9970588235294118 - } - }, - "auroc": 0.9955352328431373 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2191, - "fn": 9, - "accuracy": 0.9959090909090909 - }, - "0.01": { - "tp": 2177, - "fn": 23, - "accuracy": 0.9895454545454545 - } - }, - "auroc": 0.9878105113636363 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1187, - "fn": 13, - "accuracy": 0.9891666666666666 - }, - "0.01": { - "tp": 1167, - "fn": 33, - "accuracy": 0.9725 - } - }, - "auroc": 0.9844262152777777 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3378, - "fn": 22, - "accuracy": 0.9935294117647059 - }, - "0.01": { - "tp": 3344, - "fn": 56, - "accuracy": 0.9835294117647059 - } - }, - "auroc": 0.9866160539215687 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 4390, - "fn": 10, - "accuracy": 0.9977272727272727 - }, - "0.01": { - "tp": 4369, - "fn": 31, - "accuracy": 0.9929545454545454 - } - }, - "auroc": 0.9913779829545455 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2385, - "fn": 15, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 2365, - "fn": 35, - "accuracy": 0.9854166666666667 - } - }, - "auroc": 0.9905213541666666 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 6775, - "fn": 25, - "accuracy": 0.9963235294117647 - }, - "0.01": { - "tp": 6734, - "fn": 66, - "accuracy": 0.9902941176470588 - } - }, - "auroc": 0.9910756433823529 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999145833333333 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99930625 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996104166666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997687500000001 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.993315625 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9965421875 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998416666666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9963109375000001 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9980763020833334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9891489583333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.993325 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9912369791666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9736322916666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.970325 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9719786458333334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.981390625 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.981825 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 787, - "fn": 13, - "accuracy": 0.98375 - }, - "0.01": { - "tp": 767, - "fn": 33, - "accuracy": 0.95875 - } - }, - "auroc": 0.9816078125000001 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983708333333334 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.985471875 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9919213541666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9960927083333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.96991875 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9830057291666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9972317708333334 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9776953124999999 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 780, - "fn": 20, - "accuracy": 0.975 - } - }, - "auroc": 0.9874635416666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.958034375 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9950802083333334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9765572916666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9671135416666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.965934375 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - } - }, - "auroc": 0.9665239583333334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 348, - "fn": 52, - "accuracy": 0.87 - } - }, - "auroc": 0.9625739583333333 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9805072916666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 728, - "fn": 72, - "accuracy": 0.91 - } - }, - "auroc": 0.971540625 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.987425 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9916499999999999 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9895375000000001 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9498874999999999 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.978571875 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9642296875 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - } - }, - "auroc": 0.96865625 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9851109375 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 753, - "fn": 47, - "accuracy": 0.94125 - } - }, - "auroc": 0.97688359375 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998541666666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997791666666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998166666666666 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994510416666668 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9969927083333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998221875 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996526041666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983859375 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990192708333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9869697916666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9869697916666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9769541666666666 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9769541666666666 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9819619791666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9819619791666667 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9622916666666668 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9622916666666668 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9536552083333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9536552083333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 338, - "fn": 62, - "accuracy": 0.845 - } - }, - "auroc": 0.9579734375 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 338, - "fn": 62, - "accuracy": 0.845 - } - }, - "auroc": 0.9579734375 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998979166666666 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998979166666666 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995666666666667 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995666666666667 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997322916666668 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997322916666668 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999865625 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999865625 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.995875 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.995875 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9978703125 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9978703125 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9863458333333334 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9863458333333334 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9620302083333333 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9620302083333333 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9741880208333333 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9741880208333333 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2189, - "fn": 11, - "accuracy": 0.995 - }, - "0.01": { - "tp": 2117, - "fn": 83, - "accuracy": 0.9622727272727273 - } - }, - "auroc": 0.9880107954545455 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1192, - "fn": 8, - "accuracy": 0.9933333333333333 - } - }, - "auroc": 0.9941020833333334 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3386, - "fn": 14, - "accuracy": 0.9958823529411764 - }, - "0.01": { - "tp": 3309, - "fn": 91, - "accuracy": 0.9732352941176471 - } - }, - "auroc": 0.9901606617647059 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2169, - "fn": 31, - "accuracy": 0.985909090909091 - }, - "0.01": { - "tp": 2057, - "fn": 143, - "accuracy": 0.935 - } - }, - "auroc": 0.9794570075757576 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1179, - "fn": 21, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 1140, - "fn": 60, - "accuracy": 0.95 - } - }, - "auroc": 0.9791763888888889 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3348, - "fn": 52, - "accuracy": 0.9847058823529412 - }, - "0.01": { - "tp": 3197, - "fn": 203, - "accuracy": 0.9402941176470588 - } - }, - "auroc": 0.9793579656862744 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 4358, - "fn": 42, - "accuracy": 0.9904545454545455 - }, - "0.01": { - "tp": 4174, - "fn": 226, - "accuracy": 0.9486363636363636 - } - }, - "auroc": 0.9837339015151515 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2376, - "fn": 24, - "accuracy": 0.99 - }, - "0.01": { - "tp": 2332, - "fn": 68, - "accuracy": 0.9716666666666667 - } - }, - "auroc": 0.9866392361111112 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 6734, - "fn": 66, - "accuracy": 0.9902941176470588 - }, - "0.01": { - "tp": 6506, - "fn": 294, - "accuracy": 0.956764705882353 - } - }, - "auroc": 0.9847593137254902 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999406249999999 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997072916666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998239583333334 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999135416666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9974104166666666 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986619791666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999270833333334 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985588541666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99924296875 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9871666666666667 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.99410625 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9906364583333334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9879416666666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9717072916666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9798244791666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9875541666666667 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9829067708333332 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 778, - "fn": 22, - "accuracy": 0.9725 - } - }, - "auroc": 0.98523046875 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987625000000001 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.993540625 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9961515624999999 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981041666666666 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9791791666666666 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9886416666666666 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9984333333333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9863598958333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - } - }, - "auroc": 0.9923966145833333 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - } - }, - "auroc": 0.9536072916666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99886875 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - } - }, - "auroc": 0.9762380208333334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9819541666666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9738354166666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9778947916666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - } - }, - "auroc": 0.9677807291666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9863520833333332 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 728, - "fn": 72, - "accuracy": 0.91 - } - }, - "auroc": 0.97706640625 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9853822916666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9965510416666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9909666666666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9690083333333334 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.98378125 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9763947916666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9771953125 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9901661458333333 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 778, - "fn": 22, - "accuracy": 0.9725 - } - }, - "auroc": 0.9836807291666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997385416666666 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998604166666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997994791666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997437499999999 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9979989583333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9988713541666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997411458333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9989296875 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9993354166666666 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9922166666666666 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9922166666666666 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.98446875 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.98446875 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9883427083333334 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9883427083333334 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9804479166666666 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9804479166666666 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9687812500000001 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9687812500000001 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9746145833333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9746145833333333 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99999375 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99999375 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999880208333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999880208333334 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999479166666666 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999479166666666 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9964916666666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9964916666666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9982197916666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9982197916666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.992134375 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.992134375 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.97140625 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.97140625 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9817703124999999 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9817703124999999 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2197, - "fn": 3, - "accuracy": 0.9986363636363637 - }, - "0.01": { - "tp": 2135, - "fn": 65, - "accuracy": 0.9704545454545455 - } - }, - "auroc": 0.9899398674242424 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1195, - "fn": 5, - "accuracy": 0.9958333333333333 - } - }, - "auroc": 0.9971057291666666 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3395, - "fn": 5, - "accuracy": 0.9985294117647059 - }, - "0.01": { - "tp": 3330, - "fn": 70, - "accuracy": 0.9794117647058823 - } - }, - "auroc": 0.9924689950980392 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2177, - "fn": 23, - "accuracy": 0.9895454545454545 - }, - "0.01": { - "tp": 2131, - "fn": 69, - "accuracy": 0.9686363636363636 - } - }, - "auroc": 0.9870723484848485 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1186, - "fn": 14, - "accuracy": 0.9883333333333333 - }, - "0.01": { - "tp": 1159, - "fn": 41, - "accuracy": 0.9658333333333333 - } - }, - "auroc": 0.9839854166666666 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3363, - "fn": 37, - "accuracy": 0.9891176470588235 - }, - "0.01": { - "tp": 3290, - "fn": 110, - "accuracy": 0.9676470588235294 - } - }, - "auroc": 0.9859828431372548 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 4374, - "fn": 26, - "accuracy": 0.9940909090909091 - }, - "0.01": { - "tp": 4266, - "fn": 134, - "accuracy": 0.9695454545454546 - } - }, - "auroc": 0.9885061079545455 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2384, - "fn": 16, - "accuracy": 0.9933333333333333 - }, - "0.01": { - "tp": 2354, - "fn": 46, - "accuracy": 0.9808333333333333 - } - }, - "auroc": 0.9905455729166668 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 6758, - "fn": 42, - "accuracy": 0.9938235294117647 - }, - "0.01": { - "tp": 6620, - "fn": 180, - "accuracy": 0.9735294117647059 - } - }, - "auroc": 0.989225919117647 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980854166666666 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9968697916666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9974776041666668 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.997553125 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9890708333333333 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9933119791666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9978192708333333 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9929703125 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - } - }, - "auroc": 0.9953947916666667 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9809572916666667 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 150, - "fn": 50, - "accuracy": 0.75 - } - }, - "auroc": 0.9279958333333334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 348, - "fn": 52, - "accuracy": 0.87 - } - }, - "auroc": 0.9544765625 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.97738125 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - }, - "0.01": { - "tp": 130, - "fn": 70, - "accuracy": 0.65 - } - }, - "auroc": 0.9094270833333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 321, - "fn": 79, - "accuracy": 0.8025 - } - }, - "auroc": 0.9434041666666667 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9791692708333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - }, - "0.01": { - "tp": 280, - "fn": 120, - "accuracy": 0.7 - } - }, - "auroc": 0.9187114583333335 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 761, - "fn": 39, - "accuracy": 0.95125 - }, - "0.01": { - "tp": 669, - "fn": 131, - "accuracy": 0.83625 - } - }, - "auroc": 0.9489403645833333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9840229166666665 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.9489822916666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9665026041666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9795541666666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.9369322916666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 337, - "fn": 63, - "accuracy": 0.8425 - } - }, - "auroc": 0.9582432291666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9817885416666666 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 308, - "fn": 92, - "accuracy": 0.77 - } - }, - "auroc": 0.9429572916666666 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 778, - "fn": 22, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 691, - "fn": 109, - "accuracy": 0.86375 - } - }, - "auroc": 0.9623729166666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 156, - "fn": 44, - "accuracy": 0.78 - } - }, - "auroc": 0.9401885416666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9879489583333334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.96406875 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.975103125 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - } - }, - "auroc": 0.9358729166666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 340, - "fn": 60, - "accuracy": 0.85 - } - }, - "auroc": 0.9554880208333334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 345, - "fn": 55, - "accuracy": 0.8625 - } - }, - "auroc": 0.9576458333333333 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 347, - "fn": 53, - "accuracy": 0.8675 - } - }, - "auroc": 0.9619109375 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 776, - "fn": 24, - "accuracy": 0.97 - }, - "0.01": { - "tp": 692, - "fn": 108, - "accuracy": 0.865 - } - }, - "auroc": 0.9597783854166666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9818666666666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9676802083333333 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9747734374999999 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9737010416666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 123, - "fn": 77, - "accuracy": 0.615 - } - }, - "auroc": 0.904621875 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 313, - "fn": 87, - "accuracy": 0.7825 - } - }, - "auroc": 0.9391614583333334 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9777838541666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 306, - "fn": 94, - "accuracy": 0.765 - } - }, - "auroc": 0.9361510416666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 777, - "fn": 23, - "accuracy": 0.97125 - }, - "0.01": { - "tp": 695, - "fn": 105, - "accuracy": 0.86875 - } - }, - "auroc": 0.9569674479166668 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9953114583333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9954885416666668 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9954000000000001 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9953145833333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9920427083333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9936786458333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9953130208333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.993765625 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - } - }, - "auroc": 0.9945393229166666 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9693552083333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9693552083333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9608979166666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9608979166666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9651265625 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9651265625 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9722249999999999 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9722249999999999 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9612197916666667 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9612197916666667 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9667223958333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9667223958333333 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993270833333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993270833333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9975208333333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9975208333333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9984239583333333 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9984239583333333 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987968749999999 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987968749999999 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9962624999999999 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9962624999999999 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9975296875 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9975296875 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9778395833333333 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9778395833333333 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9657312500000002 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9657312500000002 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9717854166666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9717854166666667 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2182, - "fn": 18, - "accuracy": 0.9918181818181818 - }, - "0.01": { - "tp": 2094, - "fn": 106, - "accuracy": 0.9518181818181818 - } - }, - "auroc": 0.9816341856060606 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1181, - "fn": 19, - "accuracy": 0.9841666666666666 - }, - "0.01": { - "tp": 1090, - "fn": 110, - "accuracy": 0.9083333333333333 - } - }, - "auroc": 0.9708276041666667 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3363, - "fn": 37, - "accuracy": 0.9891176470588235 - }, - "0.01": { - "tp": 3184, - "fn": 216, - "accuracy": 0.9364705882352942 - } - }, - "auroc": 0.9778200980392157 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2172, - "fn": 28, - "accuracy": 0.9872727272727273 - }, - "0.01": { - "tp": 2073, - "fn": 127, - "accuracy": 0.9422727272727273 - } - }, - "auroc": 0.9800217803030303 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1128, - "fn": 72, - "accuracy": 0.94 - }, - "0.01": { - "tp": 944, - "fn": 256, - "accuracy": 0.7866666666666666 - } - }, - "auroc": 0.9446612847222223 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3300, - "fn": 100, - "accuracy": 0.9705882352941176 - }, - "0.01": { - "tp": 3017, - "fn": 383, - "accuracy": 0.8873529411764706 - } - }, - "auroc": 0.9675416053921568 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 4354, - "fn": 46, - "accuracy": 0.9895454545454545 - }, - "0.01": { - "tp": 4167, - "fn": 233, - "accuracy": 0.9470454545454545 - } - }, - "auroc": 0.9808279829545454 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2309, - "fn": 91, - "accuracy": 0.9620833333333333 - }, - "0.01": { - "tp": 2034, - "fn": 366, - "accuracy": 0.8475 - } - }, - "auroc": 0.9577444444444445 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 6663, - "fn": 137, - "accuracy": 0.9798529411764706 - }, - "0.01": { - "tp": 6201, - "fn": 599, - "accuracy": 0.9119117647058823 - } - }, - "auroc": 0.9726808517156863 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998177083333334 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997041666666666 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997609375000001 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998197916666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9956979166666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9977588541666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99981875 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9977010416666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9987598958333332 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9878197916666668 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.994390625 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9911052083333334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9755343750000001 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9732197916666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - } - }, - "auroc": 0.9743770833333334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9816770833333334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9838052083333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 771, - "fn": 29, - "accuracy": 0.96375 - } - }, - "auroc": 0.9827411458333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992416666666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.993025 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9961333333333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998125 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9774812499999999 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9878031249999999 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986833333333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.985253125 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - } - }, - "auroc": 0.9919682291666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9568604166666665 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9973322916666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9770963541666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.97275 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9676760416666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9702130208333333 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.9648052083333334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9825041666666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 739, - "fn": 61, - "accuracy": 0.92375 - } - }, - "auroc": 0.9736546875000001 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9864677083333333 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9943260416666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.990396875 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - } - }, - "auroc": 0.9522656250000001 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9789812499999999 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - } - }, - "auroc": 0.9656234375 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9693666666666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9866536458333334 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 760, - "fn": 40, - "accuracy": 0.95 - } - }, - "auroc": 0.97801015625 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999684375 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998770833333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997807291666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996229166666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9980510416666666 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9988369791666666 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996536458333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9989640625 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9993088541666666 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9937916666666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9937916666666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9845718749999999 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9845718749999999 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9891817708333334 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9891817708333334 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9710114583333334 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9710114583333334 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.9576895833333334 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.9576895833333334 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - } - }, - "auroc": 0.9643505208333334 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - } - }, - "auroc": 0.9643505208333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999645833333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999645833333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999645833333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999645833333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999645833333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999645833333334 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999135416666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999135416666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99639375 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99639375 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981536458333333 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981536458333333 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9898645833333333 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9898645833333333 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9675343750000001 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9675343750000001 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9786994791666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9786994791666667 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2196, - "fn": 4, - "accuracy": 0.9981818181818182 - }, - "0.01": { - "tp": 2140, - "fn": 60, - "accuracy": 0.9727272727272728 - } - }, - "auroc": 0.9894943181818181 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1196, - "fn": 4, - "accuracy": 0.9966666666666667 - } - }, - "auroc": 0.9964425347222221 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3394, - "fn": 6, - "accuracy": 0.9982352941176471 - }, - "0.01": { - "tp": 3336, - "fn": 64, - "accuracy": 0.9811764705882353 - } - }, - "auroc": 0.9919466299019608 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2166, - "fn": 34, - "accuracy": 0.9845454545454545 - }, - "0.01": { - "tp": 2092, - "fn": 108, - "accuracy": 0.9509090909090909 - } - }, - "auroc": 0.982206534090909 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1185, - "fn": 15, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 1155, - "fn": 45, - "accuracy": 0.9625 - } - }, - "auroc": 0.9818512152777777 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3351, - "fn": 49, - "accuracy": 0.9855882352941177 - }, - "0.01": { - "tp": 3247, - "fn": 153, - "accuracy": 0.955 - } - }, - "auroc": 0.9820811274509804 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 4362, - "fn": 38, - "accuracy": 0.9913636363636363 - }, - "0.01": { - "tp": 4232, - "fn": 168, - "accuracy": 0.9618181818181818 - } - }, - "auroc": 0.9858504261363636 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2383, - "fn": 17, - "accuracy": 0.9929166666666667 - }, - "0.01": { - "tp": 2351, - "fn": 49, - "accuracy": 0.9795833333333334 - } - }, - "auroc": 0.989146875 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 6745, - "fn": 55, - "accuracy": 0.9919117647058824 - }, - "0.01": { - "tp": 6583, - "fn": 217, - "accuracy": 0.9680882352941177 - } - }, - "auroc": 0.9870138786764706 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998739583333334 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99975 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998119791666666 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998354166666666 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9955177083333333 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9976765625000001 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998546875000001 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9976338541666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9987442708333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.986903125 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9942489583333334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9905760416666667 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.971515625 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9722583333333334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9718869791666667 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.979209375 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9832536458333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 769, - "fn": 31, - "accuracy": 0.96125 - } - }, - "auroc": 0.9812315104166667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993479166666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9932083333333332 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.996278125 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9981989583333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9779749999999999 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9880869791666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9987734375000001 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9855916666666666 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - } - }, - "auroc": 0.9921825520833334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9555822916666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9972041666666668 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - } - }, - "auroc": 0.9763932291666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9690979166666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9654333333333334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.967265625 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9623401041666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.98131875 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 738, - "fn": 62, - "accuracy": 0.9225 - } - }, - "auroc": 0.9718294270833333 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9855791666666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9940093750000001 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9897942708333334 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9464656250000001 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9778177083333333 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9621416666666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - } - }, - "auroc": 0.9660223958333334 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9859135416666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 756, - "fn": 44, - "accuracy": 0.945 - } - }, - "auroc": 0.97596796875 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99981875 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999145833333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998666666666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997208333333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9980385416666666 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9988796875 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997697916666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9989765625 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9993731770833334 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9907895833333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9907895833333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9842072916666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9842072916666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9874984375 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9874984375 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.969990625 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.969990625 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9571406250000001 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9571406250000001 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.963565625 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.963565625 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999833333333333 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999833333333333 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999666666666667 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999666666666667 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999750000000001 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999750000000001 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999458333333334 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999458333333334 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9961916666666666 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9961916666666666 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980687500000001 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980687500000001 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9893572916666666 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9893572916666666 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9674895833333333 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9674895833333333 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9784234375 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9784234375 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2194, - "fn": 6, - "accuracy": 0.9972727272727273 - }, - "0.01": { - "tp": 2134, - "fn": 66, - "accuracy": 0.97 - } - }, - "auroc": 0.9888338068181819 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1196, - "fn": 4, - "accuracy": 0.9966666666666667 - } - }, - "auroc": 0.9963892361111111 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3392, - "fn": 8, - "accuracy": 0.9976470588235294 - }, - "0.01": { - "tp": 3330, - "fn": 70, - "accuracy": 0.9794117647058823 - } - }, - "auroc": 0.9915004289215686 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2164, - "fn": 36, - "accuracy": 0.9836363636363636 - }, - "0.01": { - "tp": 2084, - "fn": 116, - "accuracy": 0.9472727272727273 - } - }, - "auroc": 0.9808936553030303 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1184, - "fn": 16, - "accuracy": 0.9866666666666667 - }, - "0.01": { - "tp": 1156, - "fn": 44, - "accuracy": 0.9633333333333334 - } - }, - "auroc": 0.9811734375 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3348, - "fn": 52, - "accuracy": 0.9847058823529412 - }, - "0.01": { - "tp": 3240, - "fn": 160, - "accuracy": 0.9529411764705882 - } - }, - "auroc": 0.9809924019607843 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 4358, - "fn": 42, - "accuracy": 0.9904545454545455 - }, - "0.01": { - "tp": 4218, - "fn": 182, - "accuracy": 0.9586363636363636 - } - }, - "auroc": 0.9848637310606061 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2382, - "fn": 18, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 2352, - "fn": 48, - "accuracy": 0.98 - } - }, - "auroc": 0.9887813368055556 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 6740, - "fn": 60, - "accuracy": 0.9911764705882353 - }, - "0.01": { - "tp": 6570, - "fn": 230, - "accuracy": 0.9661764705882353 - } - }, - "auroc": 0.9862464154411764 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9394302083333335 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9368520833333334 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9381411458333333 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9370197916666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9428479166666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9399338541666666 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9382250000000001 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.93985 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9390375 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9900770833333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9855218749999999 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9877994791666667 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9848864583333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9727364583333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9788114583333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9874817708333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9791291666666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 783, - "fn": 17, - "accuracy": 0.97875 - } - }, - "auroc": 0.98330546875 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9677291666666666 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9758510416666666 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9717901041666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.963021875 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9736427083333332 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9683322916666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9653755208333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9747468750000001 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9700611979166667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9581239583333334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9917822916666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - } - }, - "auroc": 0.974953125 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9873072916666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9734239583333333 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9803656249999999 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - } - }, - "auroc": 0.972715625 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.982603125 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 752, - "fn": 48, - "accuracy": 0.94 - } - }, - "auroc": 0.977659375 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9825885416666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9788635416666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9807260416666668 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9823427083333334 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9800322916666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9811875 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9824656250000001 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9794479166666665 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9809567708333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9499666666666666 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9441489583333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9470578125 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9481260416666666 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9484312500000001 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9482786458333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9490463541666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9462901041666666 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9476682291666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9824583333333332 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9824583333333332 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9784302083333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9784302083333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9804442708333334 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9804442708333334 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.971646875 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.971646875 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9656666666666667 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9656666666666667 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9686567708333335 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9686567708333335 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.936303125 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.936303125 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9369489583333332 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9369489583333332 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9366260416666666 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9366260416666666 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9337947916666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9337947916666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.94388125 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.94388125 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9388380208333333 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9388380208333333 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.96655 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.96655 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9639291666666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9639291666666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9652395833333334 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9652395833333334 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2199, - "fn": 1, - "accuracy": 0.9995454545454545 - }, - "0.01": { - "tp": 2161, - "fn": 39, - "accuracy": 0.9822727272727273 - } - }, - "auroc": 0.9616971590909089 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9688366319444445 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3397, - "fn": 3, - "accuracy": 0.9991176470588236 - }, - "0.01": { - "tp": 3359, - "fn": 41, - "accuracy": 0.9879411764705882 - } - }, - "auroc": 0.9642169730392157 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2196, - "fn": 4, - "accuracy": 0.9981818181818182 - }, - "0.01": { - "tp": 2189, - "fn": 11, - "accuracy": 0.995 - } - }, - "auroc": 0.9628691287878789 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1184, - "fn": 16, - "accuracy": 0.9866666666666667 - }, - "0.01": { - "tp": 1171, - "fn": 29, - "accuracy": 0.9758333333333333 - } - }, - "auroc": 0.9651857638888888 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3380, - "fn": 20, - "accuracy": 0.9941176470588236 - }, - "0.01": { - "tp": 3360, - "fn": 40, - "accuracy": 0.9882352941176471 - } - }, - "auroc": 0.9636867647058823 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 4395, - "fn": 5, - "accuracy": 0.9988636363636364 - }, - "0.01": { - "tp": 4350, - "fn": 50, - "accuracy": 0.9886363636363636 - } - }, - "auroc": 0.962283143939394 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2382, - "fn": 18, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 2369, - "fn": 31, - "accuracy": 0.9870833333333333 - } - }, - "auroc": 0.9670111979166667 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 6777, - "fn": 23, - "accuracy": 0.9966176470588235 - }, - "0.01": { - "tp": 6719, - "fn": 81, - "accuracy": 0.9880882352941176 - } - }, - "auroc": 0.963951868872549 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9896572916666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9891302083333333 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9893937500000001 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9887125000000001 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9859520833333333 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9873322916666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9891848958333332 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9875411458333334 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9883630208333332 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9732062499999998 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9942083333333332 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9837072916666667 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9622729166666667 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9724270833333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9673499999999999 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9677395833333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9833177083333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 779, - "fn": 21, - "accuracy": 0.97375 - } - }, - "auroc": 0.9755286458333334 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9859125 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.98519375 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.985553125 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9820666666666666 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9738625 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9779645833333335 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9839895833333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9795281250000001 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - } - }, - "auroc": 0.9817588541666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 146, - "fn": 54, - "accuracy": 0.73 - } - }, - "auroc": 0.9395968750000001 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9868510416666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 346, - "fn": 54, - "accuracy": 0.865 - } - }, - "auroc": 0.9632239583333333 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9567677083333332 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.97333125 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9650494791666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 338, - "fn": 62, - "accuracy": 0.845 - } - }, - "auroc": 0.9481822916666667 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9800911458333333 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 726, - "fn": 74, - "accuracy": 0.9075 - } - }, - "auroc": 0.9641367187500001 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9704104166666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9910947916666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9807526041666668 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9467635416666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9816385416666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9642010416666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9585869791666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9863666666666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - }, - "0.01": { - "tp": 786, - "fn": 14, - "accuracy": 0.9825 - } - }, - "auroc": 0.9724768229166667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9902416666666666 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9907333333333332 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9904875 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9895604166666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9904572916666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9900088541666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9899010416666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9905953125 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9902481770833333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.969840625 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.969840625 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9583541666666666 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9583541666666666 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9640973958333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9640973958333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9515239583333334 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9515239583333334 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9391885416666667 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9391885416666667 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9453562499999999 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9453562499999999 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.990503125 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.990503125 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9897208333333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9897208333333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9901119791666667 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9901119791666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9907989583333334 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9907989583333334 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9746166666666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9746166666666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9827078124999999 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9827078124999999 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9717635416666666 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9717635416666666 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9548697916666666 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9548697916666666 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9633166666666666 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9633166666666666 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2196, - "fn": 4, - "accuracy": 0.9981818181818182 - }, - "0.01": { - "tp": 2128, - "fn": 72, - "accuracy": 0.9672727272727273 - } - }, - "auroc": 0.9748595643939393 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1195, - "fn": 5, - "accuracy": 0.9958333333333333 - } - }, - "auroc": 0.9895352430555556 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3394, - "fn": 6, - "accuracy": 0.9982352941176471 - }, - "0.01": { - "tp": 3323, - "fn": 77, - "accuracy": 0.9773529411764705 - } - }, - "auroc": 0.9800392156862746 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2183, - "fn": 17, - "accuracy": 0.9922727272727273 - }, - "0.01": { - "tp": 2131, - "fn": 69, - "accuracy": 0.9686363636363636 - } - }, - "auroc": 0.9675357954545454 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1185, - "fn": 15, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 1161, - "fn": 39, - "accuracy": 0.9675 - } - }, - "auroc": 0.9796114583333333 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3368, - "fn": 32, - "accuracy": 0.9905882352941177 - }, - "0.01": { - "tp": 3292, - "fn": 108, - "accuracy": 0.9682352941176471 - } - }, - "auroc": 0.9717977941176471 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 4379, - "fn": 21, - "accuracy": 0.9952272727272727 - }, - "0.01": { - "tp": 4259, - "fn": 141, - "accuracy": 0.9679545454545454 - } - }, - "auroc": 0.9711976799242424 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2383, - "fn": 17, - "accuracy": 0.9929166666666667 - }, - "0.01": { - "tp": 2356, - "fn": 44, - "accuracy": 0.9816666666666667 - } - }, - "auroc": 0.9845733506944444 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 6762, - "fn": 38, - "accuracy": 0.9944117647058823 - }, - "0.01": { - "tp": 6615, - "fn": 185, - "accuracy": 0.9727941176470588 - } - }, - "auroc": 0.9759185049019607 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996791666666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995989583333333 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996390625 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99965625 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9961093750000001 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9978828125000001 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996677083333333 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9978541666666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9987609375 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9873625 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9945510416666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9909567708333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9799666666666667 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9724322916666667 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9761994791666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9836645833333334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9834916666666667 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 774, - "fn": 26, - "accuracy": 0.9675 - } - }, - "auroc": 0.983578125 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99894375 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9937906249999999 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9963671875 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9975760416666666 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.978696875 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9881364583333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9982598958333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9862437500000001 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - } - }, - "auroc": 0.9922518229166668 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9560500000000001 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9977916666666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - } - }, - "auroc": 0.9769208333333333 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9751291666666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9732833333333334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9742062499999999 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9655895833333333 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9855375 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 739, - "fn": 61, - "accuracy": 0.92375 - } - }, - "auroc": 0.9755635416666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9856083333333334 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9954447916666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9905265624999999 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.9589729166666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9828572916666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9709151041666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9722906250000001 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9891510416666666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - } - }, - "auroc": 0.9807208333333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9994895833333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992145833333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993520833333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995802083333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.997490625 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9985354166666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995348958333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9983526041666666 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.99894375 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9920333333333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9920333333333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9842135416666666 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9842135416666666 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9881234375000001 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9881234375000001 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9736093749999999 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9736093749999999 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9618458333333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9618458333333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - } - }, - "auroc": 0.9677276041666667 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 355, - "fn": 45, - "accuracy": 0.8875 - } - }, - "auroc": 0.9677276041666667 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998989583333333 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998989583333333 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995104166666666 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995104166666666 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997046875 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997046875 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998229166666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998229166666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9918541666666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9918541666666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9958385416666666 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9958385416666666 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.990609375 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.990609375 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9712770833333333 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9712770833333333 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9809432291666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9809432291666667 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2194, - "fn": 6, - "accuracy": 0.9972727272727273 - }, - "0.01": { - "tp": 2139, - "fn": 61, - "accuracy": 0.9722727272727273 - } - }, - "auroc": 0.9893733901515152 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1196, - "fn": 4, - "accuracy": 0.9966666666666667 - } - }, - "auroc": 0.9967319444444445 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3392, - "fn": 8, - "accuracy": 0.9976470588235294 - }, - "0.01": { - "tp": 3335, - "fn": 65, - "accuracy": 0.9808823529411764 - } - }, - "auroc": 0.9919705269607844 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2170, - "fn": 30, - "accuracy": 0.9863636363636363 - }, - "0.01": { - "tp": 2100, - "fn": 100, - "accuracy": 0.9545454545454546 - } - }, - "auroc": 0.9835983901515152 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1185, - "fn": 15, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 1158, - "fn": 42, - "accuracy": 0.965 - } - }, - "auroc": 0.983478298611111 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3355, - "fn": 45, - "accuracy": 0.986764705882353 - }, - "0.01": { - "tp": 3258, - "fn": 142, - "accuracy": 0.9582352941176471 - } - }, - "auroc": 0.9835560049019608 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 4364, - "fn": 36, - "accuracy": 0.9918181818181818 - }, - "0.01": { - "tp": 4239, - "fn": 161, - "accuracy": 0.9634090909090909 - } - }, - "auroc": 0.9864858901515151 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2383, - "fn": 17, - "accuracy": 0.9929166666666667 - }, - "0.01": { - "tp": 2354, - "fn": 46, - "accuracy": 0.9808333333333333 - } - }, - "auroc": 0.9901051215277779 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 6747, - "fn": 53, - "accuracy": 0.9922058823529412 - }, - "0.01": { - "tp": 6593, - "fn": 207, - "accuracy": 0.9695588235294118 - } - }, - "auroc": 0.9877632659313725 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998010416666666 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996979166666666 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997494791666667 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99979375 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9954125 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9976031249999999 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997973958333334 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9975552083333334 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9986763020833334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.98680625 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9942083333333332 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9905072916666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9720437499999999 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - } - }, - "auroc": 0.9724552083333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9722494791666666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.979425 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9833317708333333 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - } - }, - "auroc": 0.9813783854166667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992260416666667 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.993 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9961130208333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980937500000001 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9776166666666666 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9878552083333334 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986598958333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9853083333333333 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - } - }, - "auroc": 0.9919841145833334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9554854166666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9971010416666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - } - }, - "auroc": 0.9762932291666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9691489583333333 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.965540625 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9673447916666666 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9623171875000001 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9813208333333334 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 792, - "fn": 8, - "accuracy": 0.99 - }, - "0.01": { - "tp": 738, - "fn": 62, - "accuracy": 0.9225 - } - }, - "auroc": 0.9718190104166666 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9854802083333334 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9939656250000001 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9897229166666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9469208333333333 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9784104166666667 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9626656250000001 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - } - }, - "auroc": 0.9662005208333333 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9861880208333333 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 756, - "fn": 44, - "accuracy": 0.945 - } - }, - "auroc": 0.9761942708333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996781250000001 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998552083333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997666666666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996135416666667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9980364583333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9988250000000001 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996458333333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9989458333333333 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - } - }, - "auroc": 0.9992958333333334 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.990340625 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.990340625 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9841791666666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9841791666666667 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9872598958333333 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9872598958333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9698708333333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9698708333333333 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.955978125 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 172, - "fn": 28, - "accuracy": 0.86 - } - }, - "auroc": 0.955978125 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9629244791666666 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9629244791666666 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999635416666667 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999635416666667 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999645833333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999645833333334 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999640625 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999640625 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999072916666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999072916666667 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9962395833333333 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9962395833333333 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980734375 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980734375 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9891885416666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9891885416666667 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9669270833333334 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9669270833333334 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9780578125 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9780578125 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2194, - "fn": 6, - "accuracy": 0.9972727272727273 - }, - "0.01": { - "tp": 2134, - "fn": 66, - "accuracy": 0.97 - } - }, - "auroc": 0.9887043560606061 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1196, - "fn": 4, - "accuracy": 0.9966666666666667 - } - }, - "auroc": 0.9963046875 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3392, - "fn": 8, - "accuracy": 0.9976470588235294 - }, - "0.01": { - "tp": 3330, - "fn": 70, - "accuracy": 0.9794117647058823 - } - }, - "auroc": 0.9913868259803922 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2164, - "fn": 36, - "accuracy": 0.9836363636363636 - }, - "0.01": { - "tp": 2085, - "fn": 115, - "accuracy": 0.9477272727272728 - } - }, - "auroc": 0.980809375 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1184, - "fn": 16, - "accuracy": 0.9866666666666667 - }, - "0.01": { - "tp": 1156, - "fn": 44, - "accuracy": 0.9633333333333334 - } - }, - "auroc": 0.9812453125 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3348, - "fn": 52, - "accuracy": 0.9847058823529412 - }, - "0.01": { - "tp": 3241, - "fn": 159, - "accuracy": 0.9532352941176471 - } - }, - "auroc": 0.9809632352941178 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 4358, - "fn": 42, - "accuracy": 0.9904545454545455 - }, - "0.01": { - "tp": 4219, - "fn": 181, - "accuracy": 0.9588636363636364 - } - }, - "auroc": 0.9847568655303031 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2382, - "fn": 18, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 2352, - "fn": 48, - "accuracy": 0.98 - } - }, - "auroc": 0.988775 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 6740, - "fn": 60, - "accuracy": 0.9911764705882353 - }, - "0.01": { - "tp": 6571, - "fn": 229, - "accuracy": 0.9663235294117647 - } - }, - "auroc": 0.9861750306372549 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9936989583333333 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99308515625 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9933920572916666 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9932561631944444 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2385, - "fn": 15, - "accuracy": 0.99375 - } - }, - "auroc": 0.9895763020833334 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4784, - "fn": 16, - "accuracy": 0.9966666666666667 - } - }, - "auroc": 0.9914162326388889 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - } - }, - "auroc": 0.9934775607638889 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4785, - "fn": 15, - "accuracy": 0.996875 - } - }, - "auroc": 0.9913307291666666 - }, - { - "domain": "reviews", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 9584, - "fn": 16, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9924041449652776 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.98666875 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2369, - "fn": 31, - "accuracy": 0.9870833333333333 - }, - "0.01": { - "tp": 2328, - "fn": 72, - "accuracy": 0.97 - } - }, - "auroc": 0.9878283854166666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4769, - "fn": 31, - "accuracy": 0.9935416666666667 - }, - "0.01": { - "tp": 4724, - "fn": 76, - "accuracy": 0.9841666666666666 - } - }, - "auroc": 0.9872485677083334 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 2311, - "fn": 89, - "accuracy": 0.9629166666666666 - } - }, - "auroc": 0.9766861111111111 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2292, - "fn": 108, - "accuracy": 0.955 - }, - "0.01": { - "tp": 2146, - "fn": 254, - "accuracy": 0.8941666666666667 - } - }, - "auroc": 0.9668143229166666 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4688, - "fn": 112, - "accuracy": 0.9766666666666667 - }, - "0.01": { - "tp": 4457, - "fn": 343, - "accuracy": 0.9285416666666667 - } - }, - "auroc": 0.9717502170138889 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4796, - "fn": 4, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 4707, - "fn": 93, - "accuracy": 0.980625 - } - }, - "auroc": 0.9816774305555556 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4661, - "fn": 139, - "accuracy": 0.9710416666666667 - }, - "0.01": { - "tp": 4474, - "fn": 326, - "accuracy": 0.9320833333333334 - } - }, - "auroc": 0.9773213541666667 - }, - { - "domain": "reviews", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9457, - "fn": 143, - "accuracy": 0.9851041666666667 - }, - "0.01": { - "tp": 9181, - "fn": 419, - "accuracy": 0.9563541666666666 - } - }, - "auroc": 0.9794993923611112 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2393, - "fn": 7, - "accuracy": 0.9970833333333333 - } - }, - "auroc": 0.99400625 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2391, - "fn": 9, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 2348, - "fn": 52, - "accuracy": 0.9783333333333334 - } - }, - "auroc": 0.9869223090277778 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4791, - "fn": 9, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 4741, - "fn": 59, - "accuracy": 0.9877083333333333 - } - }, - "auroc": 0.9904642795138888 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2395, - "fn": 5, - "accuracy": 0.9979166666666667 - }, - "0.01": { - "tp": 2386, - "fn": 14, - "accuracy": 0.9941666666666666 - } - }, - "auroc": 0.99199296875 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2378, - "fn": 22, - "accuracy": 0.9908333333333333 - }, - "0.01": { - "tp": 2278, - "fn": 122, - "accuracy": 0.9491666666666667 - } - }, - "auroc": 0.9736358506944445 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4773, - "fn": 27, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 4664, - "fn": 136, - "accuracy": 0.9716666666666667 - } - }, - "auroc": 0.9828144097222223 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4795, - "fn": 5, - "accuracy": 0.9989583333333333 - }, - "0.01": { - "tp": 4779, - "fn": 21, - "accuracy": 0.995625 - } - }, - "auroc": 0.992999609375 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4769, - "fn": 31, - "accuracy": 0.9935416666666667 - }, - "0.01": { - "tp": 4626, - "fn": 174, - "accuracy": 0.96375 - } - }, - "auroc": 0.9802790798611111 - }, - { - "domain": "reviews", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9564, - "fn": 36, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 9405, - "fn": 195, - "accuracy": 0.9796875 - } - }, - "auroc": 0.9866393446180555 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2390, - "fn": 10, - "accuracy": 0.9958333333333333 - }, - "0.01": { - "tp": 1967, - "fn": 433, - "accuracy": 0.8195833333333333 - } - }, - "auroc": 0.95634609375 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.9952647569444444 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4790, - "fn": 10, - "accuracy": 0.9979166666666667 - }, - "0.01": { - "tp": 4363, - "fn": 437, - "accuracy": 0.9089583333333333 - } - }, - "auroc": 0.9758054253472223 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2391, - "fn": 9, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 2277, - "fn": 123, - "accuracy": 0.94875 - } - }, - "auroc": 0.9733480034722222 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2312, - "fn": 88, - "accuracy": 0.9633333333333334 - }, - "0.01": { - "tp": 2200, - "fn": 200, - "accuracy": 0.9166666666666666 - } - }, - "auroc": 0.9672430555555557 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4703, - "fn": 97, - "accuracy": 0.9797916666666666 - }, - "0.01": { - "tp": 4477, - "fn": 323, - "accuracy": 0.9327083333333334 - } - }, - "auroc": 0.9702955295138889 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4781, - "fn": 19, - "accuracy": 0.9960416666666667 - }, - "0.01": { - "tp": 4244, - "fn": 556, - "accuracy": 0.8841666666666667 - } - }, - "auroc": 0.9648470486111111 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4712, - "fn": 88, - "accuracy": 0.9816666666666667 - }, - "0.01": { - "tp": 4596, - "fn": 204, - "accuracy": 0.9575 - } - }, - "auroc": 0.9812539062500001 - }, - { - "domain": "reviews", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9493, - "fn": 107, - "accuracy": 0.9888541666666667 - }, - "0.01": { - "tp": 8840, - "fn": 760, - "accuracy": 0.9208333333333333 - } - }, - "auroc": 0.9730504774305556 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.984878125 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - }, - "0.01": { - "tp": 2372, - "fn": 28, - "accuracy": 0.9883333333333333 - } - }, - "auroc": 0.9907355902777778 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4798, - "fn": 2, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 4771, - "fn": 29, - "accuracy": 0.9939583333333334 - } - }, - "auroc": 0.9878068576388889 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2333, - "fn": 67, - "accuracy": 0.9720833333333333 - }, - "0.01": { - "tp": 2110, - "fn": 290, - "accuracy": 0.8791666666666667 - } - }, - "auroc": 0.9585163194444444 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2373, - "fn": 27, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 2279, - "fn": 121, - "accuracy": 0.9495833333333333 - } - }, - "auroc": 0.9741441840277778 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4706, - "fn": 94, - "accuracy": 0.9804166666666667 - }, - "0.01": { - "tp": 4389, - "fn": 411, - "accuracy": 0.914375 - } - }, - "auroc": 0.9663302517361112 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4733, - "fn": 67, - "accuracy": 0.9860416666666667 - }, - "0.01": { - "tp": 4509, - "fn": 291, - "accuracy": 0.939375 - } - }, - "auroc": 0.9716972222222222 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4771, - "fn": 29, - "accuracy": 0.9939583333333334 - }, - "0.01": { - "tp": 4651, - "fn": 149, - "accuracy": 0.9689583333333334 - } - }, - "auroc": 0.9824398871527779 - }, - { - "domain": "reviews", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9504, - "fn": 96, - "accuracy": 0.99 - }, - "0.01": { - "tp": 9160, - "fn": 440, - "accuracy": 0.9541666666666667 - } - }, - "auroc": 0.9770685546875 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9943717881944444 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9939145833333334 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - } - }, - "auroc": 0.9941431857638889 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9939049479166667 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2391, - "fn": 9, - "accuracy": 0.99625 - } - }, - "auroc": 0.9922289930555555 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4790, - "fn": 10, - "accuracy": 0.9979166666666667 - } - }, - "auroc": 0.9930669704861111 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4798, - "fn": 2, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9941383680555556 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4791, - "fn": 9, - "accuracy": 0.998125 - } - }, - "auroc": 0.9930717881944444 - }, - { - "domain": "reviews", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 9589, - "fn": 11, - "accuracy": 0.9988541666666667 - } - }, - "auroc": 0.9936050781250001 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2392, - "fn": 8, - "accuracy": 0.9966666666666667 - }, - "0.01": { - "tp": 2347, - "fn": 53, - "accuracy": 0.9779166666666667 - } - }, - "auroc": 0.9867511284722222 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2392, - "fn": 8, - "accuracy": 0.9966666666666667 - }, - "0.01": { - "tp": 2347, - "fn": 53, - "accuracy": 0.9779166666666667 - } - }, - "auroc": 0.9867511284722222 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2364, - "fn": 36, - "accuracy": 0.985 - }, - "0.01": { - "tp": 2286, - "fn": 114, - "accuracy": 0.9525 - } - }, - "auroc": 0.9791875 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2364, - "fn": 36, - "accuracy": 0.985 - }, - "0.01": { - "tp": 2286, - "fn": 114, - "accuracy": 0.9525 - } - }, - "auroc": 0.9791875 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4756, - "fn": 44, - "accuracy": 0.9908333333333333 - }, - "0.01": { - "tp": 4633, - "fn": 167, - "accuracy": 0.9652083333333333 - } - }, - "auroc": 0.9829693142361111 - }, - { - "domain": "reviews", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4756, - "fn": 44, - "accuracy": 0.9908333333333333 - }, - "0.01": { - "tp": 4633, - "fn": 167, - "accuracy": 0.9652083333333333 - } - }, - "auroc": 0.9829693142361111 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2357, - "fn": 43, - "accuracy": 0.9820833333333333 - }, - "0.01": { - "tp": 2212, - "fn": 188, - "accuracy": 0.9216666666666666 - } - }, - "auroc": 0.9704542534722223 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2357, - "fn": 43, - "accuracy": 0.9820833333333333 - }, - "0.01": { - "tp": 2212, - "fn": 188, - "accuracy": 0.9216666666666666 - } - }, - "auroc": 0.9704542534722223 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2300, - "fn": 100, - "accuracy": 0.9583333333333334 - }, - "0.01": { - "tp": 2134, - "fn": 266, - "accuracy": 0.8891666666666667 - } - }, - "auroc": 0.9588257812500001 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2300, - "fn": 100, - "accuracy": 0.9583333333333334 - }, - "0.01": { - "tp": 2134, - "fn": 266, - "accuracy": 0.8891666666666667 - } - }, - "auroc": 0.9588257812500001 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4657, - "fn": 143, - "accuracy": 0.9702083333333333 - }, - "0.01": { - "tp": 4346, - "fn": 454, - "accuracy": 0.9054166666666666 - } - }, - "auroc": 0.964640017361111 - }, - { - "domain": "reviews", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4657, - "fn": 143, - "accuracy": 0.9702083333333333 - }, - "0.01": { - "tp": 4346, - "fn": 454, - "accuracy": 0.9054166666666666 - } - }, - "auroc": 0.964640017361111 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9936289062499999 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9936289062499999 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2397, - "fn": 3, - "accuracy": 0.99875 - } - }, - "auroc": 0.9930896701388889 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2397, - "fn": 3, - "accuracy": 0.99875 - } - }, - "auroc": 0.9930896701388889 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4797, - "fn": 3, - "accuracy": 0.999375 - } - }, - "auroc": 0.9933592881944445 - }, - { - "domain": "reviews", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4797, - "fn": 3, - "accuracy": 0.999375 - } - }, - "auroc": 0.9933592881944445 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9933084201388889 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9933084201388889 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9886967881944445 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9886967881944445 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9910026041666666 - }, - { - "domain": "reviews", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9910026041666666 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2389, - "fn": 11, - "accuracy": 0.9954166666666666 - }, - "0.01": { - "tp": 2329, - "fn": 71, - "accuracy": 0.9704166666666667 - } - }, - "auroc": 0.98538046875 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2389, - "fn": 11, - "accuracy": 0.9954166666666666 - }, - "0.01": { - "tp": 2329, - "fn": 71, - "accuracy": 0.9704166666666667 - } - }, - "auroc": 0.98538046875 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2301, - "fn": 99, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 2190, - "fn": 210, - "accuracy": 0.9125 - } - }, - "auroc": 0.9668979166666666 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2301, - "fn": 99, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 2190, - "fn": 210, - "accuracy": 0.9125 - } - }, - "auroc": 0.9668979166666666 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4690, - "fn": 110, - "accuracy": 0.9770833333333333 - }, - "0.01": { - "tp": 4519, - "fn": 281, - "accuracy": 0.9414583333333333 - } - }, - "auroc": 0.9761391927083334 - }, - { - "domain": "reviews", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4690, - "fn": 110, - "accuracy": 0.9770833333333333 - }, - "0.01": { - "tp": 4519, - "fn": 281, - "accuracy": 0.9414583333333333 - } - }, - "auroc": 0.9761391927083334 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 26328, - "fn": 72, - "accuracy": 0.9972727272727273 - }, - "0.01": { - "tp": 25642, - "fn": 758, - "accuracy": 0.9712878787878788 - } - }, - "auroc": 0.9854084674873738 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 14358, - "fn": 42, - "accuracy": 0.9970833333333333 - }, - "0.01": { - "tp": 14244, - "fn": 156, - "accuracy": 0.9891666666666666 - } - }, - "auroc": 0.991291796875 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 40686, - "fn": 114, - "accuracy": 0.9972058823529412 - }, - "0.01": { - "tp": 39886, - "fn": 914, - "accuracy": 0.9775980392156862 - } - }, - "auroc": 0.9874849366830064 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 26080, - "fn": 320, - "accuracy": 0.9878787878787879 - }, - "0.01": { - "tp": 25289, - "fn": 1111, - "accuracy": 0.9579166666666666 - } - }, - "auroc": 0.9794911063762626 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 14155, - "fn": 245, - "accuracy": 0.9829861111111111 - }, - "0.01": { - "tp": 13679, - "fn": 721, - "accuracy": 0.9499305555555555 - } - }, - "auroc": 0.9772737847222221 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 40235, - "fn": 565, - "accuracy": 0.9861519607843138 - }, - "0.01": { - "tp": 38968, - "fn": 1832, - "accuracy": 0.9550980392156863 - } - }, - "auroc": 0.9787085222630718 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 52408, - "fn": 392, - "accuracy": 0.9925757575757576 - }, - "0.01": { - "tp": 50931, - "fn": 1869, - "accuracy": 0.9646022727272727 - } - }, - "auroc": 0.9824497869318183 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 28513, - "fn": 287, - "accuracy": 0.9900347222222222 - }, - "0.01": { - "tp": 27923, - "fn": 877, - "accuracy": 0.9695486111111111 - } - }, - "auroc": 0.9842827907986111 - }, - { - "domain": "reviews", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 80921, - "fn": 679, - "accuracy": 0.9916789215686275 - }, - "0.01": { - "tp": 78854, - "fn": 2746, - "accuracy": 0.9663480392156862 - } - }, - "auroc": 0.9830967294730392 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9977104166666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9970854166666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9973979166666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9958947916666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9862635416666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9910791666666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9968026041666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9916744791666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - } - }, - "auroc": 0.9942385416666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9651177083333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9859083333333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9755130208333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 143, - "fn": 57, - "accuracy": 0.715 - } - }, - "auroc": 0.8678729166666668 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9354843749999999 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - }, - "0.01": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - } - }, - "auroc": 0.9016786458333332 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 334, - "fn": 66, - "accuracy": 0.835 - } - }, - "auroc": 0.9164953125 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9606963541666668 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 751, - "fn": 49, - "accuracy": 0.93875 - }, - "0.01": { - "tp": 706, - "fn": 94, - "accuracy": 0.8825 - } - }, - "auroc": 0.9385958333333334 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9979979166666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9820677083333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9900328125 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9934406250000001 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9533208333333334 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9733807291666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9957192708333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9676942708333334 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - } - }, - "auroc": 0.9817067708333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9636406249999999 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9888229166666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9762317708333332 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 159, - "fn": 41, - "accuracy": 0.795 - } - }, - "auroc": 0.9145322916666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.94034375 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 347, - "fn": 53, - "accuracy": 0.8675 - } - }, - "auroc": 0.9274380208333333 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9390864583333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9645833333333335 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 781, - "fn": 19, - "accuracy": 0.97625 - }, - "0.01": { - "tp": 743, - "fn": 57, - "accuracy": 0.92875 - } - }, - "auroc": 0.9518348958333334 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9875687499999999 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9896677083333334 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9886182291666666 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - }, - "0.01": { - "tp": 145, - "fn": 55, - "accuracy": 0.725 - } - }, - "auroc": 0.8721906249999999 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9264854166666665 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - }, - "0.01": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - } - }, - "auroc": 0.8993380208333335 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - } - }, - "auroc": 0.9298796874999999 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9580765625000001 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 752, - "fn": 48, - "accuracy": 0.94 - }, - "0.01": { - "tp": 712, - "fn": 88, - "accuracy": 0.89 - } - }, - "auroc": 0.9439781249999999 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9904760416666666 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.994859375 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9926677083333333 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9745989583333333 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.9030854166666666 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9388421874999999 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9825375 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9489723958333334 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - }, - "0.01": { - "tp": 748, - "fn": 52, - "accuracy": 0.935 - } - }, - "auroc": 0.9657549479166667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9733052083333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9733052083333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9399041666666667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9399041666666667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9566046875 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9566046875 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9698895833333334 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9698895833333334 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9600104166666666 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9600104166666666 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.96495 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.96495 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999958333333333 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999958333333333 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999729166666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999729166666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999984375 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999984375 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99860625 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99860625 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9874697916666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9874697916666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9930380208333334 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9930380208333334 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9742989583333332 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9742989583333332 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.975834375 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.975834375 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9750666666666666 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9750666666666666 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2180, - "fn": 20, - "accuracy": 0.990909090909091 - }, - "0.01": { - "tp": 2153, - "fn": 47, - "accuracy": 0.9786363636363636 - } - }, - "auroc": 0.9835097537878787 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1190, - "fn": 10, - "accuracy": 0.9916666666666667 - } - }, - "auroc": 0.9897352430555556 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3377, - "fn": 23, - "accuracy": 0.9932352941176471 - }, - "0.01": { - "tp": 3343, - "fn": 57, - "accuracy": 0.9832352941176471 - } - }, - "auroc": 0.9857069852941177 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2092, - "fn": 108, - "accuracy": 0.9509090909090909 - }, - "0.01": { - "tp": 1991, - "fn": 209, - "accuracy": 0.905 - } - }, - "auroc": 0.9528838068181817 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1137, - "fn": 63, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 1090, - "fn": 110, - "accuracy": 0.9083333333333333 - } - }, - "auroc": 0.9408305555555555 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3229, - "fn": 171, - "accuracy": 0.9497058823529412 - }, - "0.01": { - "tp": 3081, - "fn": 319, - "accuracy": 0.9061764705882352 - } - }, - "auroc": 0.9486297181372549 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 4272, - "fn": 128, - "accuracy": 0.9709090909090909 - }, - "0.01": { - "tp": 4144, - "fn": 256, - "accuracy": 0.9418181818181818 - } - }, - "auroc": 0.9681967803030302 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2334, - "fn": 66, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 2280, - "fn": 120, - "accuracy": 0.95 - } - }, - "auroc": 0.9652828993055557 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 6606, - "fn": 194, - "accuracy": 0.9714705882352941 - }, - "0.01": { - "tp": 6424, - "fn": 376, - "accuracy": 0.9447058823529412 - } - }, - "auroc": 0.9671683517156864 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9977104166666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9970854166666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9973979166666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9958947916666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9862635416666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9910791666666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9968026041666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9916744791666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - } - }, - "auroc": 0.9942385416666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9651177083333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9859083333333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9755130208333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 143, - "fn": 57, - "accuracy": 0.715 - } - }, - "auroc": 0.8678729166666668 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9354843749999999 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - }, - "0.01": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - } - }, - "auroc": 0.9016786458333332 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 334, - "fn": 66, - "accuracy": 0.835 - } - }, - "auroc": 0.9164953125 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9606963541666668 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 751, - "fn": 49, - "accuracy": 0.93875 - }, - "0.01": { - "tp": 706, - "fn": 94, - "accuracy": 0.8825 - } - }, - "auroc": 0.9385958333333334 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9979979166666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9820677083333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9900328125 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9934406250000001 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9533208333333334 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9733807291666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9957192708333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9676942708333334 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - } - }, - "auroc": 0.9817067708333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9636406249999999 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9888229166666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9762317708333332 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 159, - "fn": 41, - "accuracy": 0.795 - } - }, - "auroc": 0.9145322916666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.94034375 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 347, - "fn": 53, - "accuracy": 0.8675 - } - }, - "auroc": 0.9274380208333333 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9390864583333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9645833333333335 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 781, - "fn": 19, - "accuracy": 0.97625 - }, - "0.01": { - "tp": 743, - "fn": 57, - "accuracy": 0.92875 - } - }, - "auroc": 0.9518348958333334 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9875687499999999 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9896677083333334 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9886182291666666 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - }, - "0.01": { - "tp": 145, - "fn": 55, - "accuracy": 0.725 - } - }, - "auroc": 0.8721906249999999 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9264854166666665 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - }, - "0.01": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - } - }, - "auroc": 0.8993380208333335 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - } - }, - "auroc": 0.9298796874999999 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9580765625000001 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 752, - "fn": 48, - "accuracy": 0.94 - }, - "0.01": { - "tp": 712, - "fn": 88, - "accuracy": 0.89 - } - }, - "auroc": 0.9439781249999999 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9904760416666666 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.994859375 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9926677083333333 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9745989583333333 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.9030854166666666 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9388421874999999 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9825375 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9489723958333334 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - }, - "0.01": { - "tp": 748, - "fn": 52, - "accuracy": 0.935 - } - }, - "auroc": 0.9657549479166667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9733052083333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9733052083333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9399041666666667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9399041666666667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9566046875 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9566046875 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9698895833333334 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9698895833333334 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9600104166666666 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9600104166666666 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.96495 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.96495 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999958333333333 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999958333333333 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999729166666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999729166666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999984375 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999984375 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99860625 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99860625 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9874697916666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9874697916666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9930380208333334 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9930380208333334 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9742989583333332 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9742989583333332 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.975834375 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.975834375 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9750666666666666 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9750666666666666 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2180, - "fn": 20, - "accuracy": 0.990909090909091 - }, - "0.01": { - "tp": 2153, - "fn": 47, - "accuracy": 0.9786363636363636 - } - }, - "auroc": 0.9835097537878787 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1190, - "fn": 10, - "accuracy": 0.9916666666666667 - } - }, - "auroc": 0.9897352430555556 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3377, - "fn": 23, - "accuracy": 0.9932352941176471 - }, - "0.01": { - "tp": 3343, - "fn": 57, - "accuracy": 0.9832352941176471 - } - }, - "auroc": 0.9857069852941177 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2092, - "fn": 108, - "accuracy": 0.9509090909090909 - }, - "0.01": { - "tp": 1991, - "fn": 209, - "accuracy": 0.905 - } - }, - "auroc": 0.9528838068181817 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1137, - "fn": 63, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 1090, - "fn": 110, - "accuracy": 0.9083333333333333 - } - }, - "auroc": 0.9408305555555555 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3229, - "fn": 171, - "accuracy": 0.9497058823529412 - }, - "0.01": { - "tp": 3081, - "fn": 319, - "accuracy": 0.9061764705882352 - } - }, - "auroc": 0.9486297181372549 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 4272, - "fn": 128, - "accuracy": 0.9709090909090909 - }, - "0.01": { - "tp": 4144, - "fn": 256, - "accuracy": 0.9418181818181818 - } - }, - "auroc": 0.9681967803030302 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2334, - "fn": 66, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 2280, - "fn": 120, - "accuracy": 0.95 - } - }, - "auroc": 0.9652828993055557 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 6606, - "fn": 194, - "accuracy": 0.9714705882352941 - }, - "0.01": { - "tp": 6424, - "fn": 376, - "accuracy": 0.9447058823529412 - } - }, - "auroc": 0.9671683517156864 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990145833333333 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9958208333333334 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9974177083333333 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.996559375 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9867999999999999 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9916796874999999 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9977869791666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9913104166666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9945486979166667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.981784375 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9837229166666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9827536458333335 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9400520833333332 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9312 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 362, - "fn": 38, - "accuracy": 0.905 - } - }, - "auroc": 0.9356260416666666 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - } - }, - "auroc": 0.9609182291666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9574614583333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 754, - "fn": 46, - "accuracy": 0.9425 - } - }, - "auroc": 0.95918984375 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9984072916666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9807916666666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9895994791666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99659375 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9537302083333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9751619791666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9975005208333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9672609374999999 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9823807291666666 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9920791666666666 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.990328125 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9912036458333333 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9649041666666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9772531250000001 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9710786458333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9784916666666668 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9837906249999999 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 783, - "fn": 17, - "accuracy": 0.97875 - } - }, - "auroc": 0.9811411458333333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9946552083333333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9923395833333333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9934973958333333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.946975 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.946128125 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9465515625 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9708151041666667 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9692338541666667 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 773, - "fn": 27, - "accuracy": 0.96625 - } - }, - "auroc": 0.9700244791666668 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9941677083333333 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9967947916666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9954812500000001 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9842927083333334 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.950134375 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9672135416666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9892302083333333 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9734645833333333 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 790, - "fn": 10, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 783, - "fn": 17, - "accuracy": 0.97875 - } - }, - "auroc": 0.9813473958333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9785520833333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9785520833333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9521395833333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9521395833333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9653458333333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9653458333333333 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9807416666666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9807416666666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9721791666666666 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9721791666666666 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9764604166666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9764604166666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992510416666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9992510416666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998828125 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998828125 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990395833333333 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990395833333333 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995791666666667 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995791666666667 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9876614583333332 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9876614583333332 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9936203125 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9936203125 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9811239583333333 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9811239583333333 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9799927083333334 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9799927083333334 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9805583333333333 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9805583333333333 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2184, - "fn": 16, - "accuracy": 0.9927272727272727 - }, - "0.01": { - "tp": 2181, - "fn": 19, - "accuracy": 0.9913636363636363 - } - }, - "auroc": 0.990850568181818 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1193, - "fn": 7, - "accuracy": 0.9941666666666666 - } - }, - "auroc": 0.9899663194444444 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3382, - "fn": 18, - "accuracy": 0.9947058823529412 - }, - "0.01": { - "tp": 3374, - "fn": 26, - "accuracy": 0.9923529411764705 - } - }, - "auroc": 0.9905384803921569 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2169, - "fn": 31, - "accuracy": 0.985909090909091 - }, - "0.01": { - "tp": 2140, - "fn": 60, - "accuracy": 0.9727272727272728 - } - }, - "auroc": 0.9745616477272727 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1171, - "fn": 29, - "accuracy": 0.9758333333333333 - }, - "0.01": { - "tp": 1146, - "fn": 54, - "accuracy": 0.955 - } - }, - "auroc": 0.9575409722222222 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3340, - "fn": 60, - "accuracy": 0.9823529411764705 - }, - "0.01": { - "tp": 3286, - "fn": 114, - "accuracy": 0.9664705882352941 - } - }, - "auroc": 0.9685543504901961 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 4353, - "fn": 47, - "accuracy": 0.9893181818181818 - }, - "0.01": { - "tp": 4321, - "fn": 79, - "accuracy": 0.9820454545454546 - } - }, - "auroc": 0.9827061079545454 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 2369, - "fn": 31, - "accuracy": 0.9870833333333333 - }, - "0.01": { - "tp": 2339, - "fn": 61, - "accuracy": 0.9745833333333334 - } - }, - "auroc": 0.9737536458333335 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 6722, - "fn": 78, - "accuracy": 0.9885294117647059 - }, - "0.01": { - "tp": 6660, - "fn": 140, - "accuracy": 0.9794117647058823 - } - }, - "auroc": 0.9795464154411765 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9959416666666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9939937500000001 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9949677083333333 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9950083333333333 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9766541666666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9858312499999999 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9954749999999999 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9853239583333333 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - } - }, - "auroc": 0.9903994791666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9668135416666666 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9824416666666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9746276041666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - }, - "0.01": { - "tp": 147, - "fn": 53, - "accuracy": 0.735 - } - }, - "auroc": 0.8727802083333333 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.9295010416666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - }, - "0.01": { - "tp": 321, - "fn": 79, - "accuracy": 0.8025 - } - }, - "auroc": 0.9011406249999999 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 338, - "fn": 62, - "accuracy": 0.845 - } - }, - "auroc": 0.9197968750000001 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - } - }, - "auroc": 0.9559713541666668 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 745, - "fn": 55, - "accuracy": 0.93125 - }, - "0.01": { - "tp": 709, - "fn": 91, - "accuracy": 0.88625 - } - }, - "auroc": 0.9378841145833334 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9971947916666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9754885416666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9863416666666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9932531250000001 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9462927083333332 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9697729166666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9952239583333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.960890625 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - } - }, - "auroc": 0.9780572916666668 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9657572916666666 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9840291666666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9748932291666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 154, - "fn": 46, - "accuracy": 0.77 - } - }, - "auroc": 0.8973125000000001 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9419385416666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 339, - "fn": 61, - "accuracy": 0.8475 - } - }, - "auroc": 0.9196255208333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9315348958333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9629838541666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 774, - "fn": 26, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 736, - "fn": 64, - "accuracy": 0.92 - } - }, - "auroc": 0.947259375 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9874708333333333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.986403125 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9869369791666667 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 150, - "fn": 50, - "accuracy": 0.75 - } - }, - "auroc": 0.8856083333333333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9359239583333333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 334, - "fn": 66, - "accuracy": 0.835 - } - }, - "auroc": 0.9107661458333334 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 349, - "fn": 51, - "accuracy": 0.8725 - } - }, - "auroc": 0.9365395833333333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9611635416666666 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 772, - "fn": 28, - "accuracy": 0.965 - }, - "0.01": { - "tp": 729, - "fn": 71, - "accuracy": 0.91125 - } - }, - "auroc": 0.9488515625 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.989265625 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9903885416666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - } - }, - "auroc": 0.9898270833333334 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9734010416666666 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 162, - "fn": 38, - "accuracy": 0.81 - } - }, - "auroc": 0.9144572916666666 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - } - }, - "auroc": 0.9439291666666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9813333333333334 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9524229166666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 773, - "fn": 27, - "accuracy": 0.96625 - }, - "0.01": { - "tp": 743, - "fn": 57, - "accuracy": 0.92875 - } - }, - "auroc": 0.966878125 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9599093750000001 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - } - }, - "auroc": 0.9599093750000001 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9279270833333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 168, - "fn": 32, - "accuracy": 0.84 - } - }, - "auroc": 0.9279270833333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9439182291666667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 352, - "fn": 48, - "accuracy": 0.88 - } - }, - "auroc": 0.9439182291666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.954753125 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.954753125 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.941371875 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.941371875 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - } - }, - "auroc": 0.9480624999999999 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - } - }, - "auroc": 0.9480624999999999 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999603125 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999603125 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999490625 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999490625 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995468750000001 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995468750000001 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993770833333333 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993770833333333 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9843885416666668 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9843885416666668 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9918828125 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9918828125 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.96406875 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.96406875 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.96903125 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.96903125 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - } - }, - "auroc": 0.96655 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - } - }, - "auroc": 0.96655 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2169, - "fn": 31, - "accuracy": 0.985909090909091 - }, - "0.01": { - "tp": 2142, - "fn": 58, - "accuracy": 0.9736363636363636 - } - }, - "auroc": 0.9800141098484849 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1185, - "fn": 15, - "accuracy": 0.9875 - } - }, - "auroc": 0.9854574652777778 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3366, - "fn": 34, - "accuracy": 0.99 - }, - "0.01": { - "tp": 3327, - "fn": 73, - "accuracy": 0.9785294117647059 - } - }, - "auroc": 0.9819352941176471 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2086, - "fn": 114, - "accuracy": 0.9481818181818182 - }, - "0.01": { - "tp": 1964, - "fn": 236, - "accuracy": 0.8927272727272727 - } - }, - "auroc": 0.9490520833333334 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1149, - "fn": 51, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 1096, - "fn": 104, - "accuracy": 0.9133333333333333 - } - }, - "auroc": 0.9407946180555555 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3235, - "fn": 165, - "accuracy": 0.9514705882352941 - }, - "0.01": { - "tp": 3060, - "fn": 340, - "accuracy": 0.9 - } - }, - "auroc": 0.9461376838235296 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 4255, - "fn": 145, - "accuracy": 0.9670454545454545 - }, - "0.01": { - "tp": 4106, - "fn": 294, - "accuracy": 0.9331818181818182 - } - }, - "auroc": 0.964533096590909 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2346, - "fn": 54, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 2281, - "fn": 119, - "accuracy": 0.9504166666666667 - } - }, - "auroc": 0.9631260416666667 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 6601, - "fn": 199, - "accuracy": 0.970735294117647 - }, - "0.01": { - "tp": 6387, - "fn": 413, - "accuracy": 0.9392647058823529 - } - }, - "auroc": 0.9640364889705882 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985385416666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9976468749999999 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9980927083333333 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9967666666666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9896677083333334 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9932171875 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9976526041666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9936572916666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - } - }, - "auroc": 0.9956549479166666 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.96544375 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9858020833333333 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9756229166666668 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.8991354166666666 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9349718749999999 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 340, - "fn": 60, - "accuracy": 0.85 - } - }, - "auroc": 0.9170536458333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9322895833333333 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - } - }, - "auroc": 0.9603869791666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 755, - "fn": 45, - "accuracy": 0.94375 - }, - "0.01": { - "tp": 729, - "fn": 71, - "accuracy": 0.91125 - } - }, - "auroc": 0.9463382812500001 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985843750000001 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9818989583333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9902416666666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9941166666666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.953934375 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9740255208333335 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9963505208333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9679166666666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 795, - "fn": 5, - "accuracy": 0.99375 - } - }, - "auroc": 0.9821335937500001 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9639572916666665 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9896447916666666 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9768010416666668 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.9425479166666666 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9593010416666666 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9509244791666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9532526041666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9744729166666668 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 783, - "fn": 17, - "accuracy": 0.97875 - }, - "0.01": { - "tp": 762, - "fn": 38, - "accuracy": 0.9525 - } - }, - "auroc": 0.9638627604166667 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9883916666666666 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9910625 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9897270833333334 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - }, - "0.01": { - "tp": 153, - "fn": 47, - "accuracy": 0.765 - } - }, - "auroc": 0.8996604166666666 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9347822916666666 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - }, - "0.01": { - "tp": 332, - "fn": 68, - "accuracy": 0.83 - } - }, - "auroc": 0.9172213541666667 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9440260416666667 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9629223958333334 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 764, - "fn": 36, - "accuracy": 0.955 - }, - "0.01": { - "tp": 726, - "fn": 74, - "accuracy": 0.9075 - } - }, - "auroc": 0.95347421875 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9911177083333333 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9968979166666666 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9940078124999999 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9774927083333333 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 169, - "fn": 31, - "accuracy": 0.845 - } - }, - "auroc": 0.9260489583333333 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - } - }, - "auroc": 0.9517708333333335 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9843052083333333 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - } - }, - "auroc": 0.9614734375000001 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 774, - "fn": 26, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 758, - "fn": 42, - "accuracy": 0.9475 - } - }, - "auroc": 0.9728893229166667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9754437499999999 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9754437499999999 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.938934375 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - } - }, - "auroc": 0.938934375 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9571890624999999 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9571890624999999 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9786291666666666 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9786291666666666 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9701291666666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9701291666666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9743791666666668 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9743791666666668 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999416666666666 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999416666666666 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998447916666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998447916666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998932291666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9998932291666667 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996291666666667 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9996291666666667 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9904093749999999 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9904093749999999 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9950192708333334 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9950192708333334 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.97688125 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.97688125 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9783447916666668 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9783447916666668 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9776130208333333 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.9776130208333333 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2181, - "fn": 19, - "accuracy": 0.9913636363636363 - }, - "0.01": { - "tp": 2160, - "fn": 40, - "accuracy": 0.9818181818181818 - } - }, - "auroc": 0.9851416666666666 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1190, - "fn": 10, - "accuracy": 0.9916666666666667 - } - }, - "auroc": 0.9904921875 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3378, - "fn": 22, - "accuracy": 0.9935294117647059 - }, - "0.01": { - "tp": 3350, - "fn": 50, - "accuracy": 0.9852941176470589 - } - }, - "auroc": 0.9870300857843137 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2112, - "fn": 88, - "accuracy": 0.96 - }, - "0.01": { - "tp": 2040, - "fn": 160, - "accuracy": 0.9272727272727272 - } - }, - "auroc": 0.9624892992424243 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1145, - "fn": 55, - "accuracy": 0.9541666666666667 - }, - "0.01": { - "tp": 1109, - "fn": 91, - "accuracy": 0.9241666666666667 - } - }, - "auroc": 0.9497843749999999 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3257, - "fn": 143, - "accuracy": 0.9579411764705882 - }, - "0.01": { - "tp": 3149, - "fn": 251, - "accuracy": 0.9261764705882353 - } - }, - "auroc": 0.9580052083333335 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 4293, - "fn": 107, - "accuracy": 0.9756818181818182 - }, - "0.01": { - "tp": 4200, - "fn": 200, - "accuracy": 0.9545454545454546 - } - }, - "auroc": 0.9738154829545453 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2342, - "fn": 58, - "accuracy": 0.9758333333333333 - }, - "0.01": { - "tp": 2299, - "fn": 101, - "accuracy": 0.9579166666666666 - } - }, - "auroc": 0.97013828125 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 6635, - "fn": 165, - "accuracy": 0.975735294117647 - }, - "0.01": { - "tp": 6499, - "fn": 301, - "accuracy": 0.955735294117647 - } - }, - "auroc": 0.9725176470588236 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.989190625 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9840177083333332 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9866041666666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9895302083333334 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9729083333333334 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - } - }, - "auroc": 0.9812192708333334 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9893604166666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9784630208333334 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 780, - "fn": 20, - "accuracy": 0.975 - } - }, - "auroc": 0.9839117187499999 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9606927083333332 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - }, - "0.01": { - "tp": 116, - "fn": 84, - "accuracy": 0.58 - } - }, - "auroc": 0.8297979166666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 360, - "fn": 40, - "accuracy": 0.9 - }, - "0.01": { - "tp": 304, - "fn": 96, - "accuracy": 0.76 - } - }, - "auroc": 0.8952453125 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - } - }, - "auroc": 0.9206572916666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 122, - "fn": 78, - "accuracy": 0.61 - }, - "0.01": { - "tp": 78, - "fn": 122, - "accuracy": 0.39 - } - }, - "auroc": 0.7711364583333333 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 305, - "fn": 95, - "accuracy": 0.7625 - }, - "0.01": { - "tp": 243, - "fn": 157, - "accuracy": 0.6075 - } - }, - "auroc": 0.8458968750000001 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.9406749999999999 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 286, - "fn": 114, - "accuracy": 0.715 - }, - "0.01": { - "tp": 194, - "fn": 206, - "accuracy": 0.485 - } - }, - "auroc": 0.8004671875 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 665, - "fn": 135, - "accuracy": 0.83125 - }, - "0.01": { - "tp": 547, - "fn": 253, - "accuracy": 0.68375 - } - }, - "auroc": 0.8705710937500001 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9901333333333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.955059375 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9725963541666668 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9862427083333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - }, - "0.01": { - "tp": 151, - "fn": 49, - "accuracy": 0.755 - } - }, - "auroc": 0.8951125 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 346, - "fn": 54, - "accuracy": 0.865 - } - }, - "auroc": 0.9406776041666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9881880208333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - }, - "0.01": { - "tp": 339, - "fn": 61, - "accuracy": 0.8475 - } - }, - "auroc": 0.9250859375 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 770, - "fn": 30, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 733, - "fn": 67, - "accuracy": 0.91625 - } - }, - "auroc": 0.9566369791666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9497270833333333 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9562520833333333 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9529895833333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.934746875 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 157, - "fn": 43, - "accuracy": 0.785 - } - }, - "auroc": 0.8831760416666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 335, - "fn": 65, - "accuracy": 0.8375 - } - }, - "auroc": 0.9089614583333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9422369791666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 339, - "fn": 61, - "accuracy": 0.8475 - } - }, - "auroc": 0.9197140625 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 761, - "fn": 39, - "accuracy": 0.95125 - }, - "0.01": { - "tp": 709, - "fn": 91, - "accuracy": 0.88625 - } - }, - "auroc": 0.9309755208333333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9636624999999999 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - }, - "0.01": { - "tp": 136, - "fn": 64, - "accuracy": 0.68 - } - }, - "auroc": 0.876421875 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - }, - "0.01": { - "tp": 326, - "fn": 74, - "accuracy": 0.815 - } - }, - "auroc": 0.9200421875 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 170, - "fn": 30, - "accuracy": 0.85 - }, - "0.01": { - "tp": 143, - "fn": 57, - "accuracy": 0.715 - } - }, - "auroc": 0.873440625 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 123, - "fn": 77, - "accuracy": 0.615 - }, - "0.01": { - "tp": 79, - "fn": 121, - "accuracy": 0.395 - } - }, - "auroc": 0.7784885416666667 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 293, - "fn": 107, - "accuracy": 0.7325 - }, - "0.01": { - "tp": 222, - "fn": 178, - "accuracy": 0.555 - } - }, - "auroc": 0.8259645833333333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - }, - "0.01": { - "tp": 333, - "fn": 67, - "accuracy": 0.8325 - } - }, - "auroc": 0.9185515625000001 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 294, - "fn": 106, - "accuracy": 0.735 - }, - "0.01": { - "tp": 215, - "fn": 185, - "accuracy": 0.5375 - } - }, - "auroc": 0.8274552083333334 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 661, - "fn": 139, - "accuracy": 0.82625 - }, - "0.01": { - "tp": 548, - "fn": 252, - "accuracy": 0.685 - } - }, - "auroc": 0.8730033854166668 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9889791666666665 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9816208333333333 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9853 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9641916666666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 184, - "fn": 16, - "accuracy": 0.92 - }, - "0.01": { - "tp": 156, - "fn": 44, - "accuracy": 0.78 - } - }, - "auroc": 0.9132374999999999 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 341, - "fn": 59, - "accuracy": 0.8525 - } - }, - "auroc": 0.9387145833333335 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - } - }, - "auroc": 0.9765854166666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 351, - "fn": 49, - "accuracy": 0.8775 - } - }, - "auroc": 0.9474291666666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 774, - "fn": 26, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 735, - "fn": 65, - "accuracy": 0.91875 - } - }, - "auroc": 0.9620072916666667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9396572916666666 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 176, - "fn": 24, - "accuracy": 0.88 - } - }, - "auroc": 0.9396572916666666 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.9086895833333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.9086895833333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - }, - "0.01": { - "tp": 334, - "fn": 66, - "accuracy": 0.835 - } - }, - "auroc": 0.9241734375000001 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - }, - "0.01": { - "tp": 334, - "fn": 66, - "accuracy": 0.835 - } - }, - "auroc": 0.9241734375000001 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9568635416666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9568635416666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9525604166666666 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9525604166666666 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9547119791666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9547119791666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995697916666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9995697916666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999096875 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999096875 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993333333333333 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9993333333333333 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9924104166666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9924104166666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9897885416666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9897885416666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9910994791666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9910994791666666 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.964759375 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.964759375 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.953065625 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.953065625 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9589125000000001 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9589125000000001 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2169, - "fn": 31, - "accuracy": 0.985909090909091 - }, - "0.01": { - "tp": 2115, - "fn": 85, - "accuracy": 0.9613636363636363 - } - }, - "auroc": 0.9723314393939394 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1121, - "fn": 79, - "accuracy": 0.9341666666666667 - }, - "0.01": { - "tp": 1013, - "fn": 187, - "accuracy": 0.8441666666666666 - } - }, - "auroc": 0.9305282986111112 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3290, - "fn": 110, - "accuracy": 0.9676470588235294 - }, - "0.01": { - "tp": 3128, - "fn": 272, - "accuracy": 0.92 - } - }, - "auroc": 0.9575773897058824 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2093, - "fn": 107, - "accuracy": 0.9513636363636364 - }, - "0.01": { - "tp": 1987, - "fn": 213, - "accuracy": 0.9031818181818182 - } - }, - "auroc": 0.9520009469696971 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 984, - "fn": 216, - "accuracy": 0.82 - }, - "0.01": { - "tp": 809, - "fn": 391, - "accuracy": 0.6741666666666667 - } - }, - "auroc": 0.8690098958333334 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3077, - "fn": 323, - "accuracy": 0.905 - }, - "0.01": { - "tp": 2796, - "fn": 604, - "accuracy": 0.8223529411764706 - } - }, - "auroc": 0.9227099877450982 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 4262, - "fn": 138, - "accuracy": 0.9686363636363636 - }, - "0.01": { - "tp": 4102, - "fn": 298, - "accuracy": 0.9322727272727273 - } - }, - "auroc": 0.9621661931818182 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2105, - "fn": 295, - "accuracy": 0.8770833333333333 - }, - "0.01": { - "tp": 1822, - "fn": 578, - "accuracy": 0.7591666666666667 - } - }, - "auroc": 0.8997690972222221 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 6367, - "fn": 433, - "accuracy": 0.9363235294117647 - }, - "0.01": { - "tp": 5924, - "fn": 876, - "accuracy": 0.8711764705882353 - } - }, - "auroc": 0.9401436887254901 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9997385416666668 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9982729166666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9990057291666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9983302083333334 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9922010416666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.995265625 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999034375 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9952369791666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9971356770833333 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9796979166666668 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9880281249999999 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9838630208333333 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.9373927083333333 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9382874999999999 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - } - }, - "auroc": 0.9378401041666666 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - }, - "0.01": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - } - }, - "auroc": 0.9585453125000001 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9631578125 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 767, - "fn": 33, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 745, - "fn": 55, - "accuracy": 0.93125 - } - }, - "auroc": 0.9608515624999999 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99901875 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.983359375 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9911890624999999 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99804375 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9602604166666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9791520833333334 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99853125 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9718098958333332 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9851705729166667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9745541666666666 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9935833333333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.98406875 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 174, - "fn": 26, - "accuracy": 0.87 - } - }, - "auroc": 0.94511875 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.959640625 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - } - }, - "auroc": 0.9523796875000001 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9598364583333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9766119791666668 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 786, - "fn": 14, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 761, - "fn": 39, - "accuracy": 0.95125 - } - }, - "auroc": 0.9682242187500001 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.993125 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.993496875 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9933109375 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9450218749999999 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 185, - "fn": 15, - "accuracy": 0.925 - } - }, - "auroc": 0.9454906250000001 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.94525625 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9690734375 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - } - }, - "auroc": 0.96949375 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 780, - "fn": 20, - "accuracy": 0.975 - }, - "0.01": { - "tp": 762, - "fn": 38, - "accuracy": 0.9525 - } - }, - "auroc": 0.9692835937500002 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.99666875 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.998890625 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9977796875 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9917052083333333 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.954403125 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 377, - "fn": 23, - "accuracy": 0.9425 - } - }, - "auroc": 0.9730541666666668 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9941869791666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.976646875 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 788, - "fn": 12, - "accuracy": 0.985 - }, - "0.01": { - "tp": 776, - "fn": 24, - "accuracy": 0.97 - } - }, - "auroc": 0.9854169270833333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9904625 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9904625 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9711937500000001 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9711937500000001 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.980828125 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.980828125 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9915916666666666 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9915916666666666 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9781427083333333 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9781427083333333 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9848671875 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9848671875 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999958333333333 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999958333333333 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99996875 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99996875 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999822916666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999770833333333 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999770833333333 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.992940625 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.992940625 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9964588541666667 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9964588541666667 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9893864583333334 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9893864583333334 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9878208333333334 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9878208333333334 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9886036458333334 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9886036458333334 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2188, - "fn": 12, - "accuracy": 0.9945454545454545 - }, - "0.01": { - "tp": 2180, - "fn": 20, - "accuracy": 0.990909090909091 - } - }, - "auroc": 0.9922015151515152 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1194, - "fn": 6, - "accuracy": 0.995 - } - }, - "auroc": 0.9926052083333334 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3385, - "fn": 15, - "accuracy": 0.9955882352941177 - }, - "0.01": { - "tp": 3374, - "fn": 26, - "accuracy": 0.9923529411764705 - } - }, - "auroc": 0.9923439950980393 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2155, - "fn": 45, - "accuracy": 0.9795454545454545 - }, - "0.01": { - "tp": 2105, - "fn": 95, - "accuracy": 0.9568181818181818 - } - }, - "auroc": 0.9768799242424242 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1161, - "fn": 39, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 1131, - "fn": 69, - "accuracy": 0.9425 - } - }, - "auroc": 0.9583805555555556 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3316, - "fn": 84, - "accuracy": 0.9752941176470589 - }, - "0.01": { - "tp": 3236, - "fn": 164, - "accuracy": 0.951764705882353 - } - }, - "auroc": 0.9703507352941176 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 4343, - "fn": 57, - "accuracy": 0.9870454545454546 - }, - "0.01": { - "tp": 4285, - "fn": 115, - "accuracy": 0.9738636363636364 - } - }, - "auroc": 0.9845407196969697 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2358, - "fn": 42, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 2325, - "fn": 75, - "accuracy": 0.96875 - } - }, - "auroc": 0.9754928819444445 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 6701, - "fn": 99, - "accuracy": 0.9854411764705883 - }, - "0.01": { - "tp": 6610, - "fn": 190, - "accuracy": 0.9720588235294118 - } - }, - "auroc": 0.9813473651960783 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.997725 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99734375 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.997534375 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9960031250000001 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9886229166666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9923130208333333 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9968640625 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9929833333333333 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9949236979166667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9648958333333333 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9855708333333333 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9752333333333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - }, - "0.01": { - "tp": 142, - "fn": 58, - "accuracy": 0.71 - } - }, - "auroc": 0.8594624999999998 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9357020833333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - }, - "0.01": { - "tp": 317, - "fn": 83, - "accuracy": 0.7925 - } - }, - "auroc": 0.8975822916666666 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - }, - "0.01": { - "tp": 333, - "fn": 67, - "accuracy": 0.8325 - } - }, - "auroc": 0.9121791666666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9606364583333333 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 747, - "fn": 53, - "accuracy": 0.93375 - }, - "0.01": { - "tp": 705, - "fn": 95, - "accuracy": 0.88125 - } - }, - "auroc": 0.9364078124999999 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9984083333333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9853260416666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9918671874999999 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9935385416666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.956090625 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9748145833333334 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9959734375 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9707083333333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - } - }, - "auroc": 0.9833408854166668 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9639635416666666 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9889020833333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9764328124999999 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 159, - "fn": 41, - "accuracy": 0.795 - } - }, - "auroc": 0.9138979166666665 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9400375 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 347, - "fn": 53, - "accuracy": 0.8675 - } - }, - "auroc": 0.9269677083333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9389307291666666 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9644697916666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 781, - "fn": 19, - "accuracy": 0.97625 - }, - "0.01": { - "tp": 743, - "fn": 57, - "accuracy": 0.92875 - } - }, - "auroc": 0.9517002604166668 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9876729166666667 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9889458333333333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.988309375 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - }, - "0.01": { - "tp": 144, - "fn": 56, - "accuracy": 0.72 - } - }, - "auroc": 0.8700760416666667 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9241322916666668 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - }, - "0.01": { - "tp": 317, - "fn": 83, - "accuracy": 0.7925 - } - }, - "auroc": 0.8971041666666666 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 341, - "fn": 59, - "accuracy": 0.8525 - } - }, - "auroc": 0.9288744791666668 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9565390625000001 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 751, - "fn": 49, - "accuracy": 0.93875 - }, - "0.01": { - "tp": 711, - "fn": 89, - "accuracy": 0.88875 - } - }, - "auroc": 0.9427067708333334 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9904854166666666 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9949802083333332 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9927328125000001 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9745135416666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.90776875 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - }, - "0.01": { - "tp": 353, - "fn": 47, - "accuracy": 0.8825 - } - }, - "auroc": 0.9411411458333334 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9824994791666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 361, - "fn": 39, - "accuracy": 0.9025 - } - }, - "auroc": 0.9513744791666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - }, - "0.01": { - "tp": 751, - "fn": 49, - "accuracy": 0.93875 - } - }, - "auroc": 0.9669369791666667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9731177083333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9731177083333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.94140625 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.94140625 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9572619791666668 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9572619791666668 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9702395833333334 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9702395833333334 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9612145833333334 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.9612145833333334 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9657270833333333 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9657270833333333 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999958333333333 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999958333333333 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666666 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999979166666666 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999996875 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986197916666667 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9986197916666667 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9879947916666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9879947916666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9933072916666668 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9933072916666668 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9744208333333333 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9744208333333333 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9759708333333333 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9759708333333333 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9751958333333335 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9751958333333335 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2180, - "fn": 20, - "accuracy": 0.990909090909091 - }, - "0.01": { - "tp": 2153, - "fn": 47, - "accuracy": 0.9786363636363636 - } - }, - "auroc": 0.9835949810606062 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1190, - "fn": 10, - "accuracy": 0.9916666666666667 - } - }, - "auroc": 0.9901781249999999 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3377, - "fn": 23, - "accuracy": 0.9932352941176471 - }, - "0.01": { - "tp": 3343, - "fn": 57, - "accuracy": 0.9832352941176471 - } - }, - "auroc": 0.985918443627451 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2089, - "fn": 111, - "accuracy": 0.9495454545454546 - }, - "0.01": { - "tp": 1988, - "fn": 212, - "accuracy": 0.9036363636363637 - } - }, - "auroc": 0.952188731060606 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1136, - "fn": 64, - "accuracy": 0.9466666666666667 - }, - "0.01": { - "tp": 1094, - "fn": 106, - "accuracy": 0.9116666666666666 - } - }, - "auroc": 0.9420590277777778 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3225, - "fn": 175, - "accuracy": 0.9485294117647058 - }, - "0.01": { - "tp": 3082, - "fn": 318, - "accuracy": 0.9064705882352941 - } - }, - "auroc": 0.9486135416666668 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 4269, - "fn": 131, - "accuracy": 0.9702272727272727 - }, - "0.01": { - "tp": 4141, - "fn": 259, - "accuracy": 0.9411363636363637 - } - }, - "auroc": 0.9678918560606061 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2333, - "fn": 67, - "accuracy": 0.9720833333333333 - }, - "0.01": { - "tp": 2284, - "fn": 116, - "accuracy": 0.9516666666666667 - } - }, - "auroc": 0.9661185763888889 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 6602, - "fn": 198, - "accuracy": 0.9708823529411764 - }, - "0.01": { - "tp": 6425, - "fn": 375, - "accuracy": 0.9448529411764706 - } - }, - "auroc": 0.9672659926470588 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.945940625 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9117447916666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9288427083333333 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9446677083333334 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9086260416666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.926646875 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9453041666666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9101854166666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9277447916666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9873687499999999 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9717864583333333 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9795776041666666 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9665208333333333 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - } - }, - "auroc": 0.9488854166666668 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 383, - "fn": 17, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9577031249999999 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9769447916666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - } - }, - "auroc": 0.9603359375 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 780, - "fn": 20, - "accuracy": 0.975 - }, - "0.01": { - "tp": 772, - "fn": 28, - "accuracy": 0.965 - } - }, - "auroc": 0.9686403645833334 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9448479166666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9246822916666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9347651041666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9340854166666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9514916666666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9427885416666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9394666666666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9380869791666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9387768229166668 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9778395833333333 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9891333333333333 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9834864583333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - } - }, - "auroc": 0.9788937499999999 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9692635416666666 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9740786458333333 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9783666666666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9791984375 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 789, - "fn": 11, - "accuracy": 0.98625 - } - }, - "auroc": 0.9787825520833333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9931239583333333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9469114583333333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9700177083333335 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9631260416666667 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9524479166666667 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9577869791666667 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9781249999999999 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9496796875000001 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - } - }, - "auroc": 0.9639023437500001 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9737833333333334 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9330468749999999 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - } - }, - "auroc": 0.9534151041666666 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.95310625 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9340124999999999 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.943559375 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9634447916666666 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9335296875 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 796, - "fn": 4, - "accuracy": 0.995 - }, - "0.01": { - "tp": 794, - "fn": 6, - "accuracy": 0.9925 - } - }, - "auroc": 0.9484872395833333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9830635416666667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9830635416666667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9668322916666667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.9668322916666667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9749479166666668 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9749479166666668 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9712479166666668 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9712479166666668 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.95925 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.95925 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9652489583333334 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9652489583333334 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.926115625 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.926115625 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.917 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.917 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9215578124999999 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9215578124999999 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.946290625 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.946290625 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9255020833333334 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9255020833333334 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9358963541666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9358963541666666 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.963309375 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.963309375 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9630875 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9630875 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9631984375000001 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9631984375000001 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2194, - "fn": 6, - "accuracy": 0.9972727272727273 - }, - "0.01": { - "tp": 2187, - "fn": 13, - "accuracy": 0.9940909090909091 - } - }, - "auroc": 0.9648119318181818 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1198, - "fn": 2, - "accuracy": 0.9983333333333333 - }, - "0.01": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - } - }, - "auroc": 0.9462175347222223 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3392, - "fn": 8, - "accuracy": 0.9976470588235294 - }, - "0.01": { - "tp": 3384, - "fn": 16, - "accuracy": 0.9952941176470588 - } - }, - "auroc": 0.9582492034313725 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2191, - "fn": 9, - "accuracy": 0.9959090909090909 - }, - "0.01": { - "tp": 2184, - "fn": 16, - "accuracy": 0.9927272727272727 - } - }, - "auroc": 0.9520065340909091 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1177, - "fn": 23, - "accuracy": 0.9808333333333333 - }, - "0.01": { - "tp": 1173, - "fn": 27, - "accuracy": 0.9775 - } - }, - "auroc": 0.9441211805555556 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3368, - "fn": 32, - "accuracy": 0.9905882352941177 - }, - "0.01": { - "tp": 3357, - "fn": 43, - "accuracy": 0.9873529411764705 - } - }, - "auroc": 0.9492234681372549 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 4385, - "fn": 15, - "accuracy": 0.9965909090909091 - }, - "0.01": { - "tp": 4371, - "fn": 29, - "accuracy": 0.9934090909090909 - } - }, - "auroc": 0.9584092329545455 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 2375, - "fn": 25, - "accuracy": 0.9895833333333334 - }, - "0.01": { - "tp": 2370, - "fn": 30, - "accuracy": 0.9875 - } - }, - "auroc": 0.9451693576388889 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 6760, - "fn": 40, - "accuracy": 0.9941176470588236 - }, - "0.01": { - "tp": 6741, - "fn": 59, - "accuracy": 0.9913235294117647 - } - }, - "auroc": 0.9537363357843137 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9661135416666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9708322916666665 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9684729166666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9627583333333333 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9630510416666668 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9629046874999999 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9644359375 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9669416666666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - } - }, - "auroc": 0.9656888020833333 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9378125 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9859083333333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9618604166666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.8677729166666668 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9354843749999999 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 371, - "fn": 29, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 346, - "fn": 54, - "accuracy": 0.865 - } - }, - "auroc": 0.9016286458333332 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 363, - "fn": 37, - "accuracy": 0.9075 - } - }, - "auroc": 0.9027927083333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9606963541666668 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 765, - "fn": 35, - "accuracy": 0.95625 - }, - "0.01": { - "tp": 735, - "fn": 65, - "accuracy": 0.91875 - } - }, - "auroc": 0.9317445312500001 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9813416666666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9779885416666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9796651041666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9751885416666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9523874999999999 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9637880208333334 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9782651041666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9651880208333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - } - }, - "auroc": 0.9717265625 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9431958333333333 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.960503125 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9518494791666666 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 179, - "fn": 21, - "accuracy": 0.895 - } - }, - "auroc": 0.8939864583333332 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - } - }, - "auroc": 0.9597260416666666 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - } - }, - "auroc": 0.9268562499999999 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9185911458333332 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9601145833333333 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 786, - "fn": 14, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 764, - "fn": 36, - "accuracy": 0.955 - } - }, - "auroc": 0.9393528645833333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9645364583333333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - } - }, - "auroc": 0.98590625 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - } - }, - "auroc": 0.9752213541666666 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - }, - "0.01": { - "tp": 161, - "fn": 39, - "accuracy": 0.805 - } - }, - "auroc": 0.8758666666666667 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 180, - "fn": 20, - "accuracy": 0.9 - } - }, - "auroc": 0.9359031250000001 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 341, - "fn": 59, - "accuracy": 0.8525 - } - }, - "auroc": 0.9058848958333335 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9202015625000001 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - } - }, - "auroc": 0.9609046875 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 776, - "fn": 24, - "accuracy": 0.97 - }, - "0.01": { - "tp": 734, - "fn": 66, - "accuracy": 0.9175 - } - }, - "auroc": 0.940553125 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9707625000000001 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.97658125 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.973671875 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.948353125 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9108614583333334 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9296072916666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9595578124999999 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 374, - "fn": 26, - "accuracy": 0.935 - } - }, - "auroc": 0.9437213541666668 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 783, - "fn": 17, - "accuracy": 0.97875 - }, - "0.01": { - "tp": 764, - "fn": 36, - "accuracy": 0.955 - } - }, - "auroc": 0.9516395833333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.919828125 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - } - }, - "auroc": 0.919828125 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.89110625 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - }, - "0.01": { - "tp": 171, - "fn": 29, - "accuracy": 0.855 - } - }, - "auroc": 0.89110625 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9054671875 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 384, - "fn": 16, - "accuracy": 0.96 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9054671875 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9181322916666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9181322916666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9090583333333333 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 186, - "fn": 14, - "accuracy": 0.93 - } - }, - "auroc": 0.9090583333333333 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9135953125 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9135953125 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9931531250000001 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9931531250000001 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.991796875 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.991796875 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.992475 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.992475 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9836791666666667 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9836791666666667 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9587854166666667 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9587854166666667 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9712322916666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9712322916666666 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9396979166666667 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9396979166666667 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9370218750000001 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 196, - "fn": 4, - "accuracy": 0.98 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9370218750000001 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9383598958333333 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 379, - "fn": 21, - "accuracy": 0.9475 - } - }, - "auroc": 0.9383598958333333 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2186, - "fn": 14, - "accuracy": 0.9936363636363637 - }, - "0.01": { - "tp": 2151, - "fn": 49, - "accuracy": 0.9777272727272728 - } - }, - "auroc": 0.9562048295454546 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1187, - "fn": 13, - "accuracy": 0.9891666666666666 - } - }, - "auroc": 0.9762866319444444 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3383, - "fn": 17, - "accuracy": 0.995 - }, - "0.01": { - "tp": 3338, - "fn": 62, - "accuracy": 0.981764705882353 - } - }, - "auroc": 0.963292524509804 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2143, - "fn": 57, - "accuracy": 0.9740909090909091 - }, - "0.01": { - "tp": 2045, - "fn": 155, - "accuracy": 0.9295454545454546 - } - }, - "auroc": 0.9283358901515152 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1156, - "fn": 44, - "accuracy": 0.9633333333333334 - }, - "0.01": { - "tp": 1117, - "fn": 83, - "accuracy": 0.9308333333333333 - } - }, - "auroc": 0.9429022569444445 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3299, - "fn": 101, - "accuracy": 0.9702941176470589 - }, - "0.01": { - "tp": 3162, - "fn": 238, - "accuracy": 0.93 - } - }, - "auroc": 0.9334769607843136 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 4329, - "fn": 71, - "accuracy": 0.9838636363636364 - }, - "0.01": { - "tp": 4196, - "fn": 204, - "accuracy": 0.9536363636363636 - } - }, - "auroc": 0.9422703598484847 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2353, - "fn": 47, - "accuracy": 0.9804166666666667 - }, - "0.01": { - "tp": 2304, - "fn": 96, - "accuracy": 0.96 - } - }, - "auroc": 0.9595944444444446 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 6682, - "fn": 118, - "accuracy": 0.9826470588235294 - }, - "0.01": { - "tp": 6500, - "fn": 300, - "accuracy": 0.9558823529411765 - } - }, - "auroc": 0.9483847426470589 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9985270833333333 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9967447916666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9976359374999999 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99676875 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9903645833333333 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9935666666666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9976479166666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9935546875000001 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 799, - "fn": 1, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - } - }, - "auroc": 0.9956013020833333 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9698125000000001 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9857833333333333 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - } - }, - "auroc": 0.9777979166666668 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 181, - "fn": 19, - "accuracy": 0.905 - }, - "0.01": { - "tp": 164, - "fn": 36, - "accuracy": 0.82 - } - }, - "auroc": 0.9009864583333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.937065625 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 339, - "fn": 61, - "accuracy": 0.8475 - } - }, - "auroc": 0.9190260416666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9353994791666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9614244791666668 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 757, - "fn": 43, - "accuracy": 0.94625 - }, - "0.01": { - "tp": 728, - "fn": 72, - "accuracy": 0.91 - } - }, - "auroc": 0.9484119791666668 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9984458333333334 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9807791666666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9896125 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9945854166666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9574520833333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9760187499999999 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.996515625 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9691156249999999 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 791, - "fn": 9, - "accuracy": 0.98875 - } - }, - "auroc": 0.982815625 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9634302083333333 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9907885416666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9771093750000001 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 167, - "fn": 33, - "accuracy": 0.835 - } - }, - "auroc": 0.9280083333333333 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9532760416666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 356, - "fn": 44, - "accuracy": 0.89 - } - }, - "auroc": 0.9406421875000001 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 365, - "fn": 35, - "accuracy": 0.9125 - } - }, - "auroc": 0.9457192708333333 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - } - }, - "auroc": 0.9720322916666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 784, - "fn": 16, - "accuracy": 0.98 - }, - "0.01": { - "tp": 752, - "fn": 48, - "accuracy": 0.94 - } - }, - "auroc": 0.95887578125 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9903322916666667 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9886375 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - } - }, - "auroc": 0.9894848958333333 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 177, - "fn": 23, - "accuracy": 0.885 - }, - "0.01": { - "tp": 159, - "fn": 41, - "accuracy": 0.795 - } - }, - "auroc": 0.8992708333333334 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9320906250000001 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 368, - "fn": 32, - "accuracy": 0.92 - }, - "0.01": { - "tp": 334, - "fn": 66, - "accuracy": 0.835 - } - }, - "auroc": 0.9156807291666668 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 376, - "fn": 24, - "accuracy": 0.94 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9448015625 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9603640625 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 766, - "fn": 34, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 729, - "fn": 71, - "accuracy": 0.91125 - } - }, - "auroc": 0.9525828125 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9914416666666668 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9966260416666666 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9940338541666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9797291666666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - }, - "0.01": { - "tp": 166, - "fn": 34, - "accuracy": 0.83 - } - }, - "auroc": 0.9228854166666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 373, - "fn": 27, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 359, - "fn": 41, - "accuracy": 0.8975 - } - }, - "auroc": 0.9513072916666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - }, - "0.01": { - "tp": 391, - "fn": 9, - "accuracy": 0.9775 - } - }, - "auroc": 0.9855854166666667 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 378, - "fn": 22, - "accuracy": 0.945 - }, - "0.01": { - "tp": 366, - "fn": 34, - "accuracy": 0.915 - } - }, - "auroc": 0.9597557291666668 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 772, - "fn": 28, - "accuracy": 0.965 - }, - "0.01": { - "tp": 757, - "fn": 43, - "accuracy": 0.94625 - } - }, - "auroc": 0.9726705729166667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.98145625 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.98145625 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9480177083333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 182, - "fn": 18, - "accuracy": 0.91 - } - }, - "auroc": 0.9480177083333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9647369791666667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - }, - "0.01": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - } - }, - "auroc": 0.9647369791666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9793385416666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - } - }, - "auroc": 0.9793385416666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9719010416666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9719010416666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9756197916666667 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 395, - "fn": 5, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9756197916666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999903125 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999903125 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999572916666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999572916666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999302083333333 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999302083333333 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989614583333333 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9989614583333333 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9904822916666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9904822916666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.994721875 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.994721875 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9768406249999999 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9768406249999999 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9801687499999999 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - } - }, - "auroc": 0.9801687499999999 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9785046875000001 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - } - }, - "auroc": 0.9785046875000001 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2182, - "fn": 18, - "accuracy": 0.9918181818181818 - }, - "0.01": { - "tp": 2160, - "fn": 40, - "accuracy": 0.9818181818181818 - } - }, - "auroc": 0.9862263257575756 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1189, - "fn": 11, - "accuracy": 0.9908333333333333 - } - }, - "auroc": 0.9898932291666667 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3379, - "fn": 21, - "accuracy": 0.9938235294117647 - }, - "0.01": { - "tp": 3349, - "fn": 51, - "accuracy": 0.985 - } - }, - "auroc": 0.9875205269607844 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2126, - "fn": 74, - "accuracy": 0.9663636363636363 - }, - "0.01": { - "tp": 2047, - "fn": 153, - "accuracy": 0.9304545454545454 - } - }, - "auroc": 0.9627160037878788 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1143, - "fn": 57, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 1100, - "fn": 100, - "accuracy": 0.9166666666666666 - } - }, - "auroc": 0.9488557291666666 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3269, - "fn": 131, - "accuracy": 0.9614705882352941 - }, - "0.01": { - "tp": 3147, - "fn": 253, - "accuracy": 0.9255882352941176 - } - }, - "auroc": 0.9578241421568627 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 4308, - "fn": 92, - "accuracy": 0.9790909090909091 - }, - "0.01": { - "tp": 4207, - "fn": 193, - "accuracy": 0.9561363636363637 - } - }, - "auroc": 0.9744711647727273 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2340, - "fn": 60, - "accuracy": 0.975 - }, - "0.01": { - "tp": 2289, - "fn": 111, - "accuracy": 0.95375 - } - }, - "auroc": 0.9693744791666665 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 6648, - "fn": 152, - "accuracy": 0.9776470588235294 - }, - "0.01": { - "tp": 6496, - "fn": 304, - "accuracy": 0.9552941176470588 - } - }, - "auroc": 0.9726723345588235 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9977104166666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9970854166666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9973979166666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9958947916666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9862635416666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9910791666666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9968026041666667 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9916744791666666 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 798, - "fn": 2, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - } - }, - "auroc": 0.9942385416666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 195, - "fn": 5, - "accuracy": 0.975 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.9651177083333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9859083333333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - } - }, - "auroc": 0.9755130208333334 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 143, - "fn": 57, - "accuracy": 0.715 - } - }, - "auroc": 0.8678729166666668 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 183, - "fn": 17, - "accuracy": 0.915 - }, - "0.01": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - } - }, - "auroc": 0.9354843749999999 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - }, - "0.01": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - } - }, - "auroc": 0.9016786458333332 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - }, - "0.01": { - "tp": 334, - "fn": 66, - "accuracy": 0.835 - } - }, - "auroc": 0.9164953125 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 381, - "fn": 19, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 372, - "fn": 28, - "accuracy": 0.93 - } - }, - "auroc": 0.9606963541666668 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 751, - "fn": 49, - "accuracy": 0.93875 - }, - "0.01": { - "tp": 706, - "fn": 94, - "accuracy": 0.8825 - } - }, - "auroc": 0.9385958333333334 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9979979166666666 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9820677083333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9900328125 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9934406250000001 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - } - }, - "auroc": 0.9533208333333334 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9733807291666667 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9957192708333333 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 397, - "fn": 3, - "accuracy": 0.9925 - } - }, - "auroc": 0.9676942708333334 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 797, - "fn": 3, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 793, - "fn": 7, - "accuracy": 0.99125 - } - }, - "auroc": 0.9817067708333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9636406249999999 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9888229166666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 396, - "fn": 4, - "accuracy": 0.99 - } - }, - "auroc": 0.9762317708333332 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 190, - "fn": 10, - "accuracy": 0.95 - }, - "0.01": { - "tp": 159, - "fn": 41, - "accuracy": 0.795 - } - }, - "auroc": 0.9145322916666667 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.94034375 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 382, - "fn": 18, - "accuracy": 0.955 - }, - "0.01": { - "tp": 347, - "fn": 53, - "accuracy": 0.8675 - } - }, - "auroc": 0.9274380208333333 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 389, - "fn": 11, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 357, - "fn": 43, - "accuracy": 0.8925 - } - }, - "auroc": 0.9390864583333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 392, - "fn": 8, - "accuracy": 0.98 - }, - "0.01": { - "tp": 386, - "fn": 14, - "accuracy": 0.965 - } - }, - "auroc": 0.9645833333333335 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 781, - "fn": 19, - "accuracy": 0.97625 - }, - "0.01": { - "tp": 743, - "fn": 57, - "accuracy": 0.92875 - } - }, - "auroc": 0.9518348958333334 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9875687499999999 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - } - }, - "auroc": 0.9896677083333334 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - }, - "0.01": { - "tp": 394, - "fn": 6, - "accuracy": 0.985 - } - }, - "auroc": 0.9886182291666666 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 165, - "fn": 35, - "accuracy": 0.825 - }, - "0.01": { - "tp": 145, - "fn": 55, - "accuracy": 0.725 - } - }, - "auroc": 0.8721906249999999 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - }, - "0.01": { - "tp": 173, - "fn": 27, - "accuracy": 0.865 - } - }, - "auroc": 0.9264854166666665 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 354, - "fn": 46, - "accuracy": 0.885 - }, - "0.01": { - "tp": 318, - "fn": 82, - "accuracy": 0.795 - } - }, - "auroc": 0.8993380208333335 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 364, - "fn": 36, - "accuracy": 0.91 - }, - "0.01": { - "tp": 342, - "fn": 58, - "accuracy": 0.855 - } - }, - "auroc": 0.9298796874999999 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 370, - "fn": 30, - "accuracy": 0.925 - } - }, - "auroc": 0.9580765625000001 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 752, - "fn": 48, - "accuracy": 0.94 - }, - "0.01": { - "tp": 712, - "fn": 88, - "accuracy": 0.89 - } - }, - "auroc": 0.9439781249999999 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9904760416666666 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.994859375 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9926677083333333 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9745989583333333 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 175, - "fn": 25, - "accuracy": 0.875 - }, - "0.01": { - "tp": 158, - "fn": 42, - "accuracy": 0.79 - } - }, - "auroc": 0.9030854166666666 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 369, - "fn": 31, - "accuracy": 0.9225 - }, - "0.01": { - "tp": 350, - "fn": 50, - "accuracy": 0.875 - } - }, - "auroc": 0.9388421874999999 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 393, - "fn": 7, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 390, - "fn": 10, - "accuracy": 0.975 - } - }, - "auroc": 0.9825375 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 375, - "fn": 25, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 358, - "fn": 42, - "accuracy": 0.895 - } - }, - "auroc": 0.9489723958333334 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 768, - "fn": 32, - "accuracy": 0.96 - }, - "0.01": { - "tp": 748, - "fn": 52, - "accuracy": 0.935 - } - }, - "auroc": 0.9657549479166667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9733052083333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9733052083333333 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9399041666666667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 187, - "fn": 13, - "accuracy": 0.935 - }, - "0.01": { - "tp": 178, - "fn": 22, - "accuracy": 0.89 - } - }, - "auroc": 0.9399041666666667 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9566046875 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 385, - "fn": 15, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 367, - "fn": 33, - "accuracy": 0.9175 - } - }, - "auroc": 0.9566046875 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9698895833333334 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 197, - "fn": 3, - "accuracy": 0.985 - }, - "0.01": { - "tp": 192, - "fn": 8, - "accuracy": 0.96 - } - }, - "auroc": 0.9698895833333334 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9600104166666666 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - }, - "0.01": { - "tp": 188, - "fn": 12, - "accuracy": 0.94 - } - }, - "auroc": 0.9600104166666666 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.96495 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 388, - "fn": 12, - "accuracy": 0.97 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.96495 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999958333333333 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999958333333333 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999729166666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9999729166666667 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999984375 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.999984375 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99860625 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 200, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.99860625 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9874697916666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 199, - "fn": 1, - "accuracy": 0.995 - }, - "0.01": { - "tp": 198, - "fn": 2, - "accuracy": 0.99 - } - }, - "auroc": 0.9874697916666666 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9930380208333334 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 399, - "fn": 1, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 398, - "fn": 2, - "accuracy": 0.995 - } - }, - "auroc": 0.9930380208333334 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9742989583333332 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 193, - "fn": 7, - "accuracy": 0.965 - }, - "0.01": { - "tp": 189, - "fn": 11, - "accuracy": 0.945 - } - }, - "auroc": 0.9742989583333332 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.975834375 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 194, - "fn": 6, - "accuracy": 0.97 - }, - "0.01": { - "tp": 191, - "fn": 9, - "accuracy": 0.955 - } - }, - "auroc": 0.975834375 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9750666666666666 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 387, - "fn": 13, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 380, - "fn": 20, - "accuracy": 0.95 - } - }, - "auroc": 0.9750666666666666 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2180, - "fn": 20, - "accuracy": 0.990909090909091 - }, - "0.01": { - "tp": 2153, - "fn": 47, - "accuracy": 0.9786363636363636 - } - }, - "auroc": 0.9835097537878787 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1197, - "fn": 3, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1190, - "fn": 10, - "accuracy": 0.9916666666666667 - } - }, - "auroc": 0.9897352430555556 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3377, - "fn": 23, - "accuracy": 0.9932352941176471 - }, - "0.01": { - "tp": 3343, - "fn": 57, - "accuracy": 0.9832352941176471 - } - }, - "auroc": 0.9857069852941177 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2092, - "fn": 108, - "accuracy": 0.9509090909090909 - }, - "0.01": { - "tp": 1991, - "fn": 209, - "accuracy": 0.905 - } - }, - "auroc": 0.9528838068181817 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1137, - "fn": 63, - "accuracy": 0.9475 - }, - "0.01": { - "tp": 1090, - "fn": 110, - "accuracy": 0.9083333333333333 - } - }, - "auroc": 0.9408305555555555 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3229, - "fn": 171, - "accuracy": 0.9497058823529412 - }, - "0.01": { - "tp": 3081, - "fn": 319, - "accuracy": 0.9061764705882352 - } - }, - "auroc": 0.9486297181372549 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 4272, - "fn": 128, - "accuracy": 0.9709090909090909 - }, - "0.01": { - "tp": 4144, - "fn": 256, - "accuracy": 0.9418181818181818 - } - }, - "auroc": 0.9681967803030302 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2334, - "fn": 66, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 2280, - "fn": 120, - "accuracy": 0.95 - } - }, - "auroc": 0.9652828993055557 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 6606, - "fn": 194, - "accuracy": 0.9714705882352941 - }, - "0.01": { - "tp": 6424, - "fn": 376, - "accuracy": 0.9447058823529412 - } - }, - "auroc": 0.9671683517156864 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - } - }, - "auroc": 0.9903217881944445 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2396, - "fn": 4, - "accuracy": 0.9983333333333333 - } - }, - "auroc": 0.986472829861111 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4798, - "fn": 2, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 4795, - "fn": 5, - "accuracy": 0.9989583333333333 - } - }, - "auroc": 0.9883973090277778 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2397, - "fn": 3, - "accuracy": 0.99875 - } - }, - "auroc": 0.9886730902777778 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2381, - "fn": 19, - "accuracy": 0.9920833333333333 - }, - "0.01": { - "tp": 2362, - "fn": 38, - "accuracy": 0.9841666666666666 - } - }, - "auroc": 0.9773072048611111 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4781, - "fn": 19, - "accuracy": 0.9960416666666667 - }, - "0.01": { - "tp": 4759, - "fn": 41, - "accuracy": 0.9914583333333333 - } - }, - "auroc": 0.9829901475694445 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4799, - "fn": 1, - "accuracy": 0.9997916666666666 - }, - "0.01": { - "tp": 4796, - "fn": 4, - "accuracy": 0.9991666666666666 - } - }, - "auroc": 0.9894974392361111 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4780, - "fn": 20, - "accuracy": 0.9958333333333333 - }, - "0.01": { - "tp": 4758, - "fn": 42, - "accuracy": 0.99125 - } - }, - "auroc": 0.981890017361111 - }, - { - "domain": "wiki", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9579, - "fn": 21, - "accuracy": 0.9978125 - }, - "0.01": { - "tp": 9554, - "fn": 46, - "accuracy": 0.9952083333333334 - } - }, - "auroc": 0.9856937282986111 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2346, - "fn": 54, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 2304, - "fn": 96, - "accuracy": 0.96 - } - }, - "auroc": 0.9674729166666667 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2342, - "fn": 58, - "accuracy": 0.9758333333333333 - }, - "0.01": { - "tp": 2283, - "fn": 117, - "accuracy": 0.95125 - } - }, - "auroc": 0.9713805555555557 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4688, - "fn": 112, - "accuracy": 0.9766666666666667 - }, - "0.01": { - "tp": 4587, - "fn": 213, - "accuracy": 0.955625 - } - }, - "auroc": 0.9694267361111111 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2181, - "fn": 219, - "accuracy": 0.90875 - }, - "0.01": { - "tp": 1946, - "fn": 454, - "accuracy": 0.8108333333333333 - } - }, - "auroc": 0.8973649305555557 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2135, - "fn": 265, - "accuracy": 0.8895833333333333 - }, - "0.01": { - "tp": 2009, - "fn": 391, - "accuracy": 0.8370833333333333 - } - }, - "auroc": 0.922390625 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4316, - "fn": 484, - "accuracy": 0.8991666666666667 - }, - "0.01": { - "tp": 3955, - "fn": 845, - "accuracy": 0.8239583333333333 - } - }, - "auroc": 0.9098777777777778 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4527, - "fn": 273, - "accuracy": 0.943125 - }, - "0.01": { - "tp": 4250, - "fn": 550, - "accuracy": 0.8854166666666666 - } - }, - "auroc": 0.9324189236111111 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4477, - "fn": 323, - "accuracy": 0.9327083333333334 - }, - "0.01": { - "tp": 4292, - "fn": 508, - "accuracy": 0.8941666666666667 - } - }, - "auroc": 0.9468855902777777 - }, - { - "domain": "wiki", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9004, - "fn": 596, - "accuracy": 0.9379166666666666 - }, - "0.01": { - "tp": 8542, - "fn": 1058, - "accuracy": 0.8897916666666666 - } - }, - "auroc": 0.9396522569444445 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2394, - "fn": 6, - "accuracy": 0.9975 - } - }, - "auroc": 0.9916980034722223 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2395, - "fn": 5, - "accuracy": 0.9979166666666667 - }, - "0.01": { - "tp": 2370, - "fn": 30, - "accuracy": 0.9875 - } - }, - "auroc": 0.9742980902777778 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4794, - "fn": 6, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 4764, - "fn": 36, - "accuracy": 0.9925 - } - }, - "auroc": 0.982998046875 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2378, - "fn": 22, - "accuracy": 0.9908333333333333 - }, - "0.01": { - "tp": 2373, - "fn": 27, - "accuracy": 0.98875 - } - }, - "auroc": 0.9871641493055555 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2379, - "fn": 21, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 2340, - "fn": 60, - "accuracy": 0.975 - } - }, - "auroc": 0.9488928819444444 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4757, - "fn": 43, - "accuracy": 0.9910416666666667 - }, - "0.01": { - "tp": 4713, - "fn": 87, - "accuracy": 0.981875 - } - }, - "auroc": 0.968028515625 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4777, - "fn": 23, - "accuracy": 0.9952083333333334 - }, - "0.01": { - "tp": 4767, - "fn": 33, - "accuracy": 0.993125 - } - }, - "auroc": 0.9894310763888888 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4774, - "fn": 26, - "accuracy": 0.9945833333333334 - }, - "0.01": { - "tp": 4710, - "fn": 90, - "accuracy": 0.98125 - } - }, - "auroc": 0.961595486111111 - }, - { - "domain": "wiki", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9551, - "fn": 49, - "accuracy": 0.9948958333333333 - }, - "0.01": { - "tp": 9477, - "fn": 123, - "accuracy": 0.9871875 - } - }, - "auroc": 0.97551328125 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2388, - "fn": 12, - "accuracy": 0.995 - }, - "0.01": { - "tp": 2370, - "fn": 30, - "accuracy": 0.9875 - } - }, - "auroc": 0.9654521701388888 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2392, - "fn": 8, - "accuracy": 0.9966666666666667 - }, - "0.01": { - "tp": 2364, - "fn": 36, - "accuracy": 0.985 - } - }, - "auroc": 0.9841361111111111 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4780, - "fn": 20, - "accuracy": 0.9958333333333333 - }, - "0.01": { - "tp": 4734, - "fn": 66, - "accuracy": 0.98625 - } - }, - "auroc": 0.974794140625 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2296, - "fn": 104, - "accuracy": 0.9566666666666667 - }, - "0.01": { - "tp": 2051, - "fn": 349, - "accuracy": 0.8545833333333334 - } - }, - "auroc": 0.9285844618055555 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2307, - "fn": 93, - "accuracy": 0.96125 - }, - "0.01": { - "tp": 2243, - "fn": 157, - "accuracy": 0.9345833333333333 - } - }, - "auroc": 0.9470536458333334 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4603, - "fn": 197, - "accuracy": 0.9589583333333334 - }, - "0.01": { - "tp": 4294, - "fn": 506, - "accuracy": 0.8945833333333333 - } - }, - "auroc": 0.9378190538194444 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4684, - "fn": 116, - "accuracy": 0.9758333333333333 - }, - "0.01": { - "tp": 4421, - "fn": 379, - "accuracy": 0.9210416666666666 - } - }, - "auroc": 0.9470183159722222 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4699, - "fn": 101, - "accuracy": 0.9789583333333334 - }, - "0.01": { - "tp": 4607, - "fn": 193, - "accuracy": 0.9597916666666667 - } - }, - "auroc": 0.9655948784722221 - }, - { - "domain": "wiki", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9383, - "fn": 217, - "accuracy": 0.9773958333333334 - }, - "0.01": { - "tp": 9028, - "fn": 572, - "accuracy": 0.9404166666666667 - } - }, - "auroc": 0.9563065972222223 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2386, - "fn": 14, - "accuracy": 0.9941666666666666 - }, - "0.01": { - "tp": 2365, - "fn": 35, - "accuracy": 0.9854166666666667 - } - }, - "auroc": 0.9854730902777777 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2362, - "fn": 38, - "accuracy": 0.9841666666666666 - }, - "0.01": { - "tp": 2307, - "fn": 93, - "accuracy": 0.96125 - } - }, - "auroc": 0.9765940104166666 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4748, - "fn": 52, - "accuracy": 0.9891666666666666 - }, - "0.01": { - "tp": 4672, - "fn": 128, - "accuracy": 0.9733333333333334 - } - }, - "auroc": 0.9810335503472223 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2136, - "fn": 264, - "accuracy": 0.89 - }, - "0.01": { - "tp": 1912, - "fn": 488, - "accuracy": 0.7966666666666666 - } - }, - "auroc": 0.897968142361111 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2232, - "fn": 168, - "accuracy": 0.93 - }, - "0.01": { - "tp": 2060, - "fn": 340, - "accuracy": 0.8583333333333333 - } - }, - "auroc": 0.9220703125 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4368, - "fn": 432, - "accuracy": 0.91 - }, - "0.01": { - "tp": 3972, - "fn": 828, - "accuracy": 0.8275 - } - }, - "auroc": 0.9100192274305555 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4522, - "fn": 278, - "accuracy": 0.9420833333333334 - }, - "0.01": { - "tp": 4277, - "fn": 523, - "accuracy": 0.8910416666666666 - } - }, - "auroc": 0.9417206163194444 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4594, - "fn": 206, - "accuracy": 0.9570833333333333 - }, - "0.01": { - "tp": 4367, - "fn": 433, - "accuracy": 0.9097916666666667 - } - }, - "auroc": 0.9493321614583332 - }, - { - "domain": "wiki", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9116, - "fn": 484, - "accuracy": 0.9495833333333333 - }, - "0.01": { - "tp": 8644, - "fn": 956, - "accuracy": 0.9004166666666666 - } - }, - "auroc": 0.9455263888888888 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2387, - "fn": 13, - "accuracy": 0.9945833333333334 - }, - "0.01": { - "tp": 2379, - "fn": 21, - "accuracy": 0.99125 - } - }, - "auroc": 0.988175 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2388, - "fn": 12, - "accuracy": 0.995 - } - }, - "auroc": 0.9875337673611111 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4787, - "fn": 13, - "accuracy": 0.9972916666666667 - }, - "0.01": { - "tp": 4767, - "fn": 33, - "accuracy": 0.993125 - } - }, - "auroc": 0.9878543836805556 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2336, - "fn": 64, - "accuracy": 0.9733333333333334 - }, - "0.01": { - "tp": 2307, - "fn": 93, - "accuracy": 0.96125 - } - }, - "auroc": 0.9725485243055555 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2199, - "fn": 201, - "accuracy": 0.91625 - }, - "0.01": { - "tp": 2031, - "fn": 369, - "accuracy": 0.84625 - } - }, - "auroc": 0.92025546875 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4535, - "fn": 265, - "accuracy": 0.9447916666666667 - }, - "0.01": { - "tp": 4338, - "fn": 462, - "accuracy": 0.90375 - } - }, - "auroc": 0.9464019965277777 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4723, - "fn": 77, - "accuracy": 0.9839583333333334 - }, - "0.01": { - "tp": 4686, - "fn": 114, - "accuracy": 0.97625 - } - }, - "auroc": 0.9803617621527777 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4599, - "fn": 201, - "accuracy": 0.958125 - }, - "0.01": { - "tp": 4419, - "fn": 381, - "accuracy": 0.920625 - } - }, - "auroc": 0.9538946180555556 - }, - { - "domain": "wiki", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 9322, - "fn": 278, - "accuracy": 0.9710416666666667 - }, - "0.01": { - "tp": 9105, - "fn": 495, - "accuracy": 0.9484375 - } - }, - "auroc": 0.9671281901041666 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2368, - "fn": 32, - "accuracy": 0.9866666666666667 - }, - "0.01": { - "tp": 2278, - "fn": 122, - "accuracy": 0.9491666666666667 - } - }, - "auroc": 0.9684505208333334 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2368, - "fn": 32, - "accuracy": 0.9866666666666667 - }, - "0.01": { - "tp": 2278, - "fn": 122, - "accuracy": 0.9491666666666667 - } - }, - "auroc": 0.9684505208333334 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2255, - "fn": 145, - "accuracy": 0.9395833333333333 - }, - "0.01": { - "tp": 2141, - "fn": 259, - "accuracy": 0.8920833333333333 - } - }, - "auroc": 0.9388299479166666 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2255, - "fn": 145, - "accuracy": 0.9395833333333333 - }, - "0.01": { - "tp": 2141, - "fn": 259, - "accuracy": 0.8920833333333333 - } - }, - "auroc": 0.9388299479166666 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4623, - "fn": 177, - "accuracy": 0.963125 - }, - "0.01": { - "tp": 4419, - "fn": 381, - "accuracy": 0.920625 - } - }, - "auroc": 0.9536402343749999 - }, - { - "domain": "wiki", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4623, - "fn": 177, - "accuracy": 0.963125 - }, - "0.01": { - "tp": 4419, - "fn": 381, - "accuracy": 0.920625 - } - }, - "auroc": 0.9536402343749999 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2369, - "fn": 31, - "accuracy": 0.9870833333333333 - }, - "0.01": { - "tp": 2317, - "fn": 83, - "accuracy": 0.9654166666666667 - } - }, - "auroc": 0.9676005208333331 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2369, - "fn": 31, - "accuracy": 0.9870833333333333 - }, - "0.01": { - "tp": 2317, - "fn": 83, - "accuracy": 0.9654166666666667 - } - }, - "auroc": 0.9676005208333331 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2324, - "fn": 76, - "accuracy": 0.9683333333333334 - }, - "0.01": { - "tp": 2267, - "fn": 133, - "accuracy": 0.9445833333333333 - } - }, - "auroc": 0.9579865451388889 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2324, - "fn": 76, - "accuracy": 0.9683333333333334 - }, - "0.01": { - "tp": 2267, - "fn": 133, - "accuracy": 0.9445833333333333 - } - }, - "auroc": 0.9579865451388889 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4693, - "fn": 107, - "accuracy": 0.9777083333333333 - }, - "0.01": { - "tp": 4584, - "fn": 216, - "accuracy": 0.955 - } - }, - "auroc": 0.9627935329861113 - }, - { - "domain": "wiki", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4693, - "fn": 107, - "accuracy": 0.9777083333333333 - }, - "0.01": { - "tp": 4584, - "fn": 216, - "accuracy": 0.955 - } - }, - "auroc": 0.9627935329861113 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9931263888888888 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9931263888888888 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9921583333333334 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 2400, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9921583333333334 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9926423611111111 - }, - { - "domain": "wiki", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 4800, - "fn": 0, - "accuracy": 1.0 - } - }, - "auroc": 0.9926423611111111 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - } - }, - "auroc": 0.9928618923611111 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2399, - "fn": 1, - "accuracy": 0.9995833333333334 - }, - "0.01": { - "tp": 2398, - "fn": 2, - "accuracy": 0.9991666666666666 - } - }, - "auroc": 0.9928618923611111 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2394, - "fn": 6, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 2386, - "fn": 14, - "accuracy": 0.9941666666666666 - } - }, - "auroc": 0.9808635416666667 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2394, - "fn": 6, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 2386, - "fn": 14, - "accuracy": 0.9941666666666666 - } - }, - "auroc": 0.9808635416666667 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4793, - "fn": 7, - "accuracy": 0.9985416666666667 - }, - "0.01": { - "tp": 4784, - "fn": 16, - "accuracy": 0.9966666666666667 - } - }, - "auroc": 0.9868627170138888 - }, - { - "domain": "wiki", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4793, - "fn": 7, - "accuracy": 0.9985416666666667 - }, - "0.01": { - "tp": 4784, - "fn": 16, - "accuracy": 0.9966666666666667 - } - }, - "auroc": 0.9868627170138888 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2332, - "fn": 68, - "accuracy": 0.9716666666666667 - }, - "0.01": { - "tp": 2284, - "fn": 116, - "accuracy": 0.9516666666666667 - } - }, - "auroc": 0.9711154513888889 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2332, - "fn": 68, - "accuracy": 0.9716666666666667 - }, - "0.01": { - "tp": 2284, - "fn": 116, - "accuracy": 0.9516666666666667 - } - }, - "auroc": 0.9711154513888889 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2340, - "fn": 60, - "accuracy": 0.975 - }, - "0.01": { - "tp": 2293, - "fn": 107, - "accuracy": 0.9554166666666667 - } - }, - "auroc": 0.9710006076388888 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 2340, - "fn": 60, - "accuracy": 0.975 - }, - "0.01": { - "tp": 2293, - "fn": 107, - "accuracy": 0.9554166666666667 - } - }, - "auroc": 0.9710006076388888 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4672, - "fn": 128, - "accuracy": 0.9733333333333334 - }, - "0.01": { - "tp": 4577, - "fn": 223, - "accuracy": 0.9535416666666666 - } - }, - "auroc": 0.9710580295138889 - }, - { - "domain": "wiki", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 4672, - "fn": 128, - "accuracy": 0.9733333333333334 - }, - "0.01": { - "tp": 4577, - "fn": 223, - "accuracy": 0.9535416666666666 - } - }, - "auroc": 0.9710580295138889 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 26173, - "fn": 227, - "accuracy": 0.9914015151515152 - }, - "0.01": { - "tp": 25888, - "fn": 512, - "accuracy": 0.9806060606060606 - } - }, - "auroc": 0.9801588857323232 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 14290, - "fn": 110, - "accuracy": 0.9923611111111111 - }, - "0.01": { - "tp": 14108, - "fn": 292, - "accuracy": 0.9797222222222223 - } - }, - "auroc": 0.9800692274305555 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 40463, - "fn": 337, - "accuracy": 0.9917401960784313 - }, - "0.01": { - "tp": 39996, - "fn": 804, - "accuracy": 0.9802941176470589 - } - }, - "auroc": 0.9801272416258169 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 25440, - "fn": 960, - "accuracy": 0.9636363636363636 - }, - "0.01": { - "tp": 24473, - "fn": 1927, - "accuracy": 0.9270075757575758 - } - }, - "auroc": 0.9557402067550504 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 13633, - "fn": 767, - "accuracy": 0.9467361111111111 - }, - "0.01": { - "tp": 13045, - "fn": 1355, - "accuracy": 0.9059027777777777 - } - }, - "auroc": 0.9396616898148149 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 39073, - "fn": 1727, - "accuracy": 0.957671568627451 - }, - "0.01": { - "tp": 37518, - "fn": 3282, - "accuracy": 0.9195588235294118 - } - }, - "auroc": 0.9500654360702615 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 51613, - "fn": 1187, - "accuracy": 0.9775189393939394 - }, - "0.01": { - "tp": 50361, - "fn": 2439, - "accuracy": 0.9538068181818182 - } - }, - "auroc": 0.9679495462436869 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 27923, - "fn": 877, - "accuracy": 0.9695486111111111 - }, - "0.01": { - "tp": 27153, - "fn": 1647, - "accuracy": 0.9428125 - } - }, - "auroc": 0.9598654586226852 - }, - { - "domain": "wiki", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 79536, - "fn": 2064, - "accuracy": 0.9747058823529412 - }, - "0.01": { - "tp": 77514, - "fn": 4086, - "accuracy": 0.9499264705882353 - } - }, - "auroc": 0.9650963388480392 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - } - }, - "auroc": 0.9965804524739583 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - } - }, - "auroc": 0.995021923828125 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3198, - "fn": 2, - "accuracy": 0.999375 - } - }, - "auroc": 0.9958011881510417 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - } - }, - "auroc": 0.995738916015625 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1580, - "fn": 20, - "accuracy": 0.9875 - } - }, - "auroc": 0.9898985351562499 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3197, - "fn": 3, - "accuracy": 0.9990625 - }, - "0.01": { - "tp": 3177, - "fn": 23, - "accuracy": 0.9928125 - } - }, - "auroc": 0.9928187255859375 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3196, - "fn": 4, - "accuracy": 0.99875 - } - }, - "auroc": 0.9961596842447917 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3197, - "fn": 3, - "accuracy": 0.9990625 - }, - "0.01": { - "tp": 3179, - "fn": 21, - "accuracy": 0.9934375 - } - }, - "auroc": 0.9924602294921875 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 6397, - "fn": 3, - "accuracy": 0.99953125 - }, - "0.01": { - "tp": 6375, - "fn": 25, - "accuracy": 0.99609375 - } - }, - "auroc": 0.9943099568684897 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1564, - "fn": 36, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 1491, - "fn": 109, - "accuracy": 0.931875 - } - }, - "auroc": 0.972055859375 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - }, - "0.01": { - "tp": 1554, - "fn": 46, - "accuracy": 0.97125 - } - }, - "auroc": 0.9870034830729166 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3140, - "fn": 60, - "accuracy": 0.98125 - }, - "0.01": { - "tp": 3045, - "fn": 155, - "accuracy": 0.9515625 - } - }, - "auroc": 0.9795296712239583 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1462, - "fn": 138, - "accuracy": 0.91375 - }, - "0.01": { - "tp": 1299, - "fn": 301, - "accuracy": 0.811875 - } - }, - "auroc": 0.9275415201822916 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1534, - "fn": 66, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 1483, - "fn": 117, - "accuracy": 0.926875 - } - }, - "auroc": 0.9657855468750001 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2996, - "fn": 204, - "accuracy": 0.93625 - }, - "0.01": { - "tp": 2782, - "fn": 418, - "accuracy": 0.869375 - } - }, - "auroc": 0.9466635335286457 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3026, - "fn": 174, - "accuracy": 0.945625 - }, - "0.01": { - "tp": 2790, - "fn": 410, - "accuracy": 0.871875 - } - }, - "auroc": 0.9497986897786458 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3110, - "fn": 90, - "accuracy": 0.971875 - }, - "0.01": { - "tp": 3037, - "fn": 163, - "accuracy": 0.9490625 - } - }, - "auroc": 0.9763945149739582 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 6136, - "fn": 264, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 5827, - "fn": 573, - "accuracy": 0.91046875 - } - }, - "auroc": 0.9630966023763021 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - } - }, - "auroc": 0.9928874837239583 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 1577, - "fn": 23, - "accuracy": 0.985625 - } - }, - "auroc": 0.9868442545572917 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3181, - "fn": 19, - "accuracy": 0.9940625 - }, - "0.01": { - "tp": 3161, - "fn": 39, - "accuracy": 0.9878125 - } - }, - "auroc": 0.989865869140625 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1588, - "fn": 12, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - } - }, - "auroc": 0.990519140625 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - }, - "0.01": { - "tp": 1524, - "fn": 76, - "accuracy": 0.9525 - } - }, - "auroc": 0.9697653483072917 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3172, - "fn": 28, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 3100, - "fn": 100, - "accuracy": 0.96875 - } - }, - "auroc": 0.9801422444661458 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3179, - "fn": 21, - "accuracy": 0.9934375 - }, - "0.01": { - "tp": 3160, - "fn": 40, - "accuracy": 0.9875 - } - }, - "auroc": 0.9917033121744792 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3174, - "fn": 26, - "accuracy": 0.991875 - }, - "0.01": { - "tp": 3101, - "fn": 99, - "accuracy": 0.9690625 - } - }, - "auroc": 0.9783048014322916 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 6353, - "fn": 47, - "accuracy": 0.99265625 - }, - "0.01": { - "tp": 6261, - "fn": 139, - "accuracy": 0.97828125 - } - }, - "auroc": 0.9850040568033854 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1518, - "fn": 82, - "accuracy": 0.94875 - }, - "0.01": { - "tp": 1403, - "fn": 197, - "accuracy": 0.876875 - } - }, - "auroc": 0.952021923828125 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 1571, - "fn": 29, - "accuracy": 0.981875 - } - }, - "auroc": 0.991476220703125 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3113, - "fn": 87, - "accuracy": 0.9728125 - }, - "0.01": { - "tp": 2974, - "fn": 226, - "accuracy": 0.929375 - } - }, - "auroc": 0.971749072265625 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1512, - "fn": 88, - "accuracy": 0.945 - }, - "0.01": { - "tp": 1285, - "fn": 315, - "accuracy": 0.803125 - } - }, - "auroc": 0.9330303873697916 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1551, - "fn": 49, - "accuracy": 0.969375 - }, - "0.01": { - "tp": 1488, - "fn": 112, - "accuracy": 0.93 - } - }, - "auroc": 0.9527174479166667 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3063, - "fn": 137, - "accuracy": 0.9571875 - }, - "0.01": { - "tp": 2773, - "fn": 427, - "accuracy": 0.8665625 - } - }, - "auroc": 0.9428739176432291 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3030, - "fn": 170, - "accuracy": 0.946875 - }, - "0.01": { - "tp": 2688, - "fn": 512, - "accuracy": 0.84 - } - }, - "auroc": 0.9425261555989584 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3146, - "fn": 54, - "accuracy": 0.983125 - }, - "0.01": { - "tp": 3059, - "fn": 141, - "accuracy": 0.9559375 - } - }, - "auroc": 0.9720968343098957 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 6176, - "fn": 224, - "accuracy": 0.965 - }, - "0.01": { - "tp": 5747, - "fn": 653, - "accuracy": 0.89796875 - } - }, - "auroc": 0.957311494954427 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1565, - "fn": 35, - "accuracy": 0.978125 - }, - "0.01": { - "tp": 1502, - "fn": 98, - "accuracy": 0.93875 - } - }, - "auroc": 0.9784849772135416 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1572, - "fn": 28, - "accuracy": 0.9825 - } - }, - "auroc": 0.9847144856770833 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3154, - "fn": 46, - "accuracy": 0.985625 - }, - "0.01": { - "tp": 3074, - "fn": 126, - "accuracy": 0.960625 - } - }, - "auroc": 0.9815997314453125 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1361, - "fn": 239, - "accuracy": 0.850625 - }, - "0.01": { - "tp": 1129, - "fn": 471, - "accuracy": 0.705625 - } - }, - "auroc": 0.897958544921875 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1534, - "fn": 66, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 1433, - "fn": 167, - "accuracy": 0.895625 - } - }, - "auroc": 0.9491408528645833 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2895, - "fn": 305, - "accuracy": 0.9046875 - }, - "0.01": { - "tp": 2562, - "fn": 638, - "accuracy": 0.800625 - } - }, - "auroc": 0.923549698893229 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2926, - "fn": 274, - "accuracy": 0.914375 - }, - "0.01": { - "tp": 2631, - "fn": 569, - "accuracy": 0.8221875 - } - }, - "auroc": 0.9382217610677083 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3123, - "fn": 77, - "accuracy": 0.9759375 - }, - "0.01": { - "tp": 3005, - "fn": 195, - "accuracy": 0.9390625 - } - }, - "auroc": 0.9669276692708333 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 6049, - "fn": 351, - "accuracy": 0.94515625 - }, - "0.01": { - "tp": 5636, - "fn": 764, - "accuracy": 0.880625 - } - }, - "auroc": 0.952574715169271 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.99486494140625 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1593, - "fn": 7, - "accuracy": 0.995625 - } - }, - "auroc": 0.9939198567708333 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3193, - "fn": 7, - "accuracy": 0.9978125 - }, - "0.01": { - "tp": 3188, - "fn": 12, - "accuracy": 0.99625 - } - }, - "auroc": 0.9943923990885417 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1588, - "fn": 12, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 1582, - "fn": 18, - "accuracy": 0.98875 - } - }, - "auroc": 0.9908359863281251 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1564, - "fn": 36, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 1522, - "fn": 78, - "accuracy": 0.95125 - } - }, - "auroc": 0.974113525390625 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3152, - "fn": 48, - "accuracy": 0.985 - }, - "0.01": { - "tp": 3104, - "fn": 96, - "accuracy": 0.97 - } - }, - "auroc": 0.9824747558593749 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3184, - "fn": 16, - "accuracy": 0.995 - }, - "0.01": { - "tp": 3177, - "fn": 23, - "accuracy": 0.9928125 - } - }, - "auroc": 0.9928504638671874 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3161, - "fn": 39, - "accuracy": 0.9878125 - }, - "0.01": { - "tp": 3115, - "fn": 85, - "accuracy": 0.9734375 - } - }, - "auroc": 0.9840166910807292 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 6345, - "fn": 55, - "accuracy": 0.99140625 - }, - "0.01": { - "tp": 6292, - "fn": 108, - "accuracy": 0.983125 - } - }, - "auroc": 0.9884335774739583 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1548, - "fn": 52, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 1490, - "fn": 110, - "accuracy": 0.93125 - } - }, - "auroc": 0.9677334309895833 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1548, - "fn": 52, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 1490, - "fn": 110, - "accuracy": 0.93125 - } - }, - "auroc": 0.9677334309895833 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1527, - "fn": 73, - "accuracy": 0.954375 - }, - "0.01": { - "tp": 1450, - "fn": 150, - "accuracy": 0.90625 - } - }, - "auroc": 0.9576980305989583 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1527, - "fn": 73, - "accuracy": 0.954375 - }, - "0.01": { - "tp": 1450, - "fn": 150, - "accuracy": 0.90625 - } - }, - "auroc": 0.9576980305989583 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3075, - "fn": 125, - "accuracy": 0.9609375 - }, - "0.01": { - "tp": 2940, - "fn": 260, - "accuracy": 0.91875 - } - }, - "auroc": 0.9627157307942708 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3075, - "fn": 125, - "accuracy": 0.9609375 - }, - "0.01": { - "tp": 2940, - "fn": 260, - "accuracy": 0.91875 - } - }, - "auroc": 0.9627157307942708 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1484, - "fn": 116, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 1346, - "fn": 254, - "accuracy": 0.84125 - } - }, - "auroc": 0.9343791015625 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1484, - "fn": 116, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 1346, - "fn": 254, - "accuracy": 0.84125 - } - }, - "auroc": 0.9343791015625 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1401, - "fn": 199, - "accuracy": 0.875625 - }, - "0.01": { - "tp": 1225, - "fn": 375, - "accuracy": 0.765625 - } - }, - "auroc": 0.9109469889322916 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1401, - "fn": 199, - "accuracy": 0.875625 - }, - "0.01": { - "tp": 1225, - "fn": 375, - "accuracy": 0.765625 - } - }, - "auroc": 0.9109469889322916 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2885, - "fn": 315, - "accuracy": 0.9015625 - }, - "0.01": { - "tp": 2571, - "fn": 629, - "accuracy": 0.8034375 - } - }, - "auroc": 0.9226630452473958 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2885, - "fn": 315, - "accuracy": 0.9015625 - }, - "0.01": { - "tp": 2571, - "fn": 629, - "accuracy": 0.8034375 - } - }, - "auroc": 0.9226630452473958 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9924631184895834 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9924631184895834 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - } - }, - "auroc": 0.990256982421875 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - } - }, - "auroc": 0.990256982421875 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3187, - "fn": 13, - "accuracy": 0.9959375 - } - }, - "auroc": 0.9913600504557292 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3187, - "fn": 13, - "accuracy": 0.9959375 - } - }, - "auroc": 0.9913600504557292 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9935093912760417 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9935093912760417 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1552, - "fn": 48, - "accuracy": 0.97 - } - }, - "auroc": 0.9767893391927083 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1552, - "fn": 48, - "accuracy": 0.97 - } - }, - "auroc": 0.9767893391927083 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3186, - "fn": 14, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 3147, - "fn": 53, - "accuracy": 0.9834375 - } - }, - "auroc": 0.985149365234375 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 3186, - "fn": 14, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 3147, - "fn": 53, - "accuracy": 0.9834375 - } - }, - "auroc": 0.985149365234375 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1521, - "fn": 79, - "accuracy": 0.950625 - }, - "0.01": { - "tp": 1448, - "fn": 152, - "accuracy": 0.905 - } - }, - "auroc": 0.9640538248697916 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1521, - "fn": 79, - "accuracy": 0.950625 - }, - "0.01": { - "tp": 1448, - "fn": 152, - "accuracy": 0.905 - } - }, - "auroc": 0.9640538248697916 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1468, - "fn": 132, - "accuracy": 0.9175 - }, - "0.01": { - "tp": 1371, - "fn": 229, - "accuracy": 0.856875 - } - }, - "auroc": 0.9452570963541667 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 1468, - "fn": 132, - "accuracy": 0.9175 - }, - "0.01": { - "tp": 1371, - "fn": 229, - "accuracy": 0.856875 - } - }, - "auroc": 0.9452570963541667 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2989, - "fn": 211, - "accuracy": 0.9340625 - }, - "0.01": { - "tp": 2819, - "fn": 381, - "accuracy": 0.8809375 - } - }, - "auroc": 0.9546554606119791 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 2989, - "fn": 211, - "accuracy": 0.9340625 - }, - "0.01": { - "tp": 2819, - "fn": 381, - "accuracy": 0.8809375 - } - }, - "auroc": 0.9546554606119791 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 17180, - "fn": 420, - "accuracy": 0.9761363636363637 - }, - "0.01": { - "tp": 16649, - "fn": 951, - "accuracy": 0.945965909090909 - } - }, - "auroc": 0.9762758641098486 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 9547, - "fn": 53, - "accuracy": 0.9944791666666667 - }, - "0.01": { - "tp": 9466, - "fn": 134, - "accuracy": 0.9860416666666667 - } - }, - "auroc": 0.9898300374348958 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 26727, - "fn": 473, - "accuracy": 0.9826102941176471 - }, - "0.01": { - "tp": 26115, - "fn": 1085, - "accuracy": 0.960110294117647 - } - }, - "auroc": 0.981059689989277 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 16690, - "fn": 910, - "accuracy": 0.9482954545454545 - }, - "0.01": { - "tp": 15657, - "fn": 1943, - "accuracy": 0.8896022727272728 - } - }, - "auroc": 0.9560520848129734 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 9364, - "fn": 236, - "accuracy": 0.9754166666666667 - }, - "0.01": { - "tp": 9030, - "fn": 570, - "accuracy": 0.940625 - } - }, - "auroc": 0.9669035427517362 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 26054, - "fn": 1146, - "accuracy": 0.9578676470588235 - }, - "0.01": { - "tp": 24687, - "fn": 2513, - "accuracy": 0.907610294117647 - } - }, - "auroc": 0.9598820111443014 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 33870, - "fn": 1330, - "accuracy": 0.962215909090909 - }, - "0.01": { - "tp": 32306, - "fn": 2894, - "accuracy": 0.9177840909090909 - } - }, - "auroc": 0.966163974461411 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 18911, - "fn": 289, - "accuracy": 0.9849479166666667 - }, - "0.01": { - "tp": 18496, - "fn": 704, - "accuracy": 0.9633333333333334 - } - }, - "auroc": 0.9783667900933161 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "none", - "accuracy": { - "0.05": { - "tp": 52781, - "fn": 1619, - "accuracy": 0.9702389705882353 - }, - "0.01": { - "tp": 50802, - "fn": 3598, - "accuracy": 0.933860294117647 - } - }, - "auroc": 0.9704708505667892 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - } - }, - "auroc": 0.9965804524739583 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - } - }, - "auroc": 0.995021923828125 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3198, - "fn": 2, - "accuracy": 0.999375 - } - }, - "auroc": 0.9958011881510417 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - } - }, - "auroc": 0.995738916015625 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1580, - "fn": 20, - "accuracy": 0.9875 - } - }, - "auroc": 0.9898985351562499 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3197, - "fn": 3, - "accuracy": 0.9990625 - }, - "0.01": { - "tp": 3177, - "fn": 23, - "accuracy": 0.9928125 - } - }, - "auroc": 0.9928187255859375 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3196, - "fn": 4, - "accuracy": 0.99875 - } - }, - "auroc": 0.9961596842447917 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3197, - "fn": 3, - "accuracy": 0.9990625 - }, - "0.01": { - "tp": 3179, - "fn": 21, - "accuracy": 0.9934375 - } - }, - "auroc": 0.9924602294921875 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 6397, - "fn": 3, - "accuracy": 0.99953125 - }, - "0.01": { - "tp": 6375, - "fn": 25, - "accuracy": 0.99609375 - } - }, - "auroc": 0.9943099568684897 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1564, - "fn": 36, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 1491, - "fn": 109, - "accuracy": 0.931875 - } - }, - "auroc": 0.972055859375 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - }, - "0.01": { - "tp": 1554, - "fn": 46, - "accuracy": 0.97125 - } - }, - "auroc": 0.9870034830729166 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3140, - "fn": 60, - "accuracy": 0.98125 - }, - "0.01": { - "tp": 3045, - "fn": 155, - "accuracy": 0.9515625 - } - }, - "auroc": 0.9795296712239583 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1462, - "fn": 138, - "accuracy": 0.91375 - }, - "0.01": { - "tp": 1299, - "fn": 301, - "accuracy": 0.811875 - } - }, - "auroc": 0.9275415201822916 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1534, - "fn": 66, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 1483, - "fn": 117, - "accuracy": 0.926875 - } - }, - "auroc": 0.9657855468750001 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2996, - "fn": 204, - "accuracy": 0.93625 - }, - "0.01": { - "tp": 2782, - "fn": 418, - "accuracy": 0.869375 - } - }, - "auroc": 0.9466635335286457 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3026, - "fn": 174, - "accuracy": 0.945625 - }, - "0.01": { - "tp": 2790, - "fn": 410, - "accuracy": 0.871875 - } - }, - "auroc": 0.9497986897786458 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3110, - "fn": 90, - "accuracy": 0.971875 - }, - "0.01": { - "tp": 3037, - "fn": 163, - "accuracy": 0.9490625 - } - }, - "auroc": 0.9763945149739582 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 6136, - "fn": 264, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 5827, - "fn": 573, - "accuracy": 0.91046875 - } - }, - "auroc": 0.9630966023763021 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - } - }, - "auroc": 0.9928874837239583 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 1577, - "fn": 23, - "accuracy": 0.985625 - } - }, - "auroc": 0.9868442545572917 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3181, - "fn": 19, - "accuracy": 0.9940625 - }, - "0.01": { - "tp": 3161, - "fn": 39, - "accuracy": 0.9878125 - } - }, - "auroc": 0.989865869140625 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1588, - "fn": 12, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - } - }, - "auroc": 0.990519140625 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - }, - "0.01": { - "tp": 1524, - "fn": 76, - "accuracy": 0.9525 - } - }, - "auroc": 0.9697653483072917 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3172, - "fn": 28, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 3100, - "fn": 100, - "accuracy": 0.96875 - } - }, - "auroc": 0.9801422444661458 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3179, - "fn": 21, - "accuracy": 0.9934375 - }, - "0.01": { - "tp": 3160, - "fn": 40, - "accuracy": 0.9875 - } - }, - "auroc": 0.9917033121744792 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3174, - "fn": 26, - "accuracy": 0.991875 - }, - "0.01": { - "tp": 3101, - "fn": 99, - "accuracy": 0.9690625 - } - }, - "auroc": 0.9783048014322916 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 6353, - "fn": 47, - "accuracy": 0.99265625 - }, - "0.01": { - "tp": 6261, - "fn": 139, - "accuracy": 0.97828125 - } - }, - "auroc": 0.9850040568033854 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1518, - "fn": 82, - "accuracy": 0.94875 - }, - "0.01": { - "tp": 1403, - "fn": 197, - "accuracy": 0.876875 - } - }, - "auroc": 0.952021923828125 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 1571, - "fn": 29, - "accuracy": 0.981875 - } - }, - "auroc": 0.991476220703125 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3113, - "fn": 87, - "accuracy": 0.9728125 - }, - "0.01": { - "tp": 2974, - "fn": 226, - "accuracy": 0.929375 - } - }, - "auroc": 0.971749072265625 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1512, - "fn": 88, - "accuracy": 0.945 - }, - "0.01": { - "tp": 1285, - "fn": 315, - "accuracy": 0.803125 - } - }, - "auroc": 0.9330303873697916 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1551, - "fn": 49, - "accuracy": 0.969375 - }, - "0.01": { - "tp": 1488, - "fn": 112, - "accuracy": 0.93 - } - }, - "auroc": 0.9527174479166667 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3063, - "fn": 137, - "accuracy": 0.9571875 - }, - "0.01": { - "tp": 2773, - "fn": 427, - "accuracy": 0.8665625 - } - }, - "auroc": 0.9428739176432291 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3030, - "fn": 170, - "accuracy": 0.946875 - }, - "0.01": { - "tp": 2688, - "fn": 512, - "accuracy": 0.84 - } - }, - "auroc": 0.9425261555989584 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3146, - "fn": 54, - "accuracy": 0.983125 - }, - "0.01": { - "tp": 3059, - "fn": 141, - "accuracy": 0.9559375 - } - }, - "auroc": 0.9720968343098957 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 6176, - "fn": 224, - "accuracy": 0.965 - }, - "0.01": { - "tp": 5747, - "fn": 653, - "accuracy": 0.89796875 - } - }, - "auroc": 0.957311494954427 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1565, - "fn": 35, - "accuracy": 0.978125 - }, - "0.01": { - "tp": 1502, - "fn": 98, - "accuracy": 0.93875 - } - }, - "auroc": 0.9784849772135416 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1572, - "fn": 28, - "accuracy": 0.9825 - } - }, - "auroc": 0.9847144856770833 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3154, - "fn": 46, - "accuracy": 0.985625 - }, - "0.01": { - "tp": 3074, - "fn": 126, - "accuracy": 0.960625 - } - }, - "auroc": 0.9815997314453125 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1361, - "fn": 239, - "accuracy": 0.850625 - }, - "0.01": { - "tp": 1129, - "fn": 471, - "accuracy": 0.705625 - } - }, - "auroc": 0.897958544921875 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1534, - "fn": 66, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 1433, - "fn": 167, - "accuracy": 0.895625 - } - }, - "auroc": 0.949140869140625 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2895, - "fn": 305, - "accuracy": 0.9046875 - }, - "0.01": { - "tp": 2562, - "fn": 638, - "accuracy": 0.800625 - } - }, - "auroc": 0.92354970703125 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2926, - "fn": 274, - "accuracy": 0.914375 - }, - "0.01": { - "tp": 2631, - "fn": 569, - "accuracy": 0.8221875 - } - }, - "auroc": 0.9382217610677083 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3123, - "fn": 77, - "accuracy": 0.9759375 - }, - "0.01": { - "tp": 3005, - "fn": 195, - "accuracy": 0.9390625 - } - }, - "auroc": 0.9669276774088542 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 6049, - "fn": 351, - "accuracy": 0.94515625 - }, - "0.01": { - "tp": 5636, - "fn": 764, - "accuracy": 0.880625 - } - }, - "auroc": 0.9525747192382813 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.99486494140625 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1593, - "fn": 7, - "accuracy": 0.995625 - } - }, - "auroc": 0.9939198567708333 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3193, - "fn": 7, - "accuracy": 0.9978125 - }, - "0.01": { - "tp": 3188, - "fn": 12, - "accuracy": 0.99625 - } - }, - "auroc": 0.9943923990885417 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1588, - "fn": 12, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 1582, - "fn": 18, - "accuracy": 0.98875 - } - }, - "auroc": 0.9908359863281251 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1564, - "fn": 36, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 1522, - "fn": 78, - "accuracy": 0.95125 - } - }, - "auroc": 0.974113525390625 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3152, - "fn": 48, - "accuracy": 0.985 - }, - "0.01": { - "tp": 3104, - "fn": 96, - "accuracy": 0.97 - } - }, - "auroc": 0.9824747558593749 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3184, - "fn": 16, - "accuracy": 0.995 - }, - "0.01": { - "tp": 3177, - "fn": 23, - "accuracy": 0.9928125 - } - }, - "auroc": 0.9928504638671874 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3161, - "fn": 39, - "accuracy": 0.9878125 - }, - "0.01": { - "tp": 3115, - "fn": 85, - "accuracy": 0.9734375 - } - }, - "auroc": 0.9840166910807292 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 6345, - "fn": 55, - "accuracy": 0.99140625 - }, - "0.01": { - "tp": 6292, - "fn": 108, - "accuracy": 0.983125 - } - }, - "auroc": 0.9884335774739583 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1548, - "fn": 52, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 1490, - "fn": 110, - "accuracy": 0.93125 - } - }, - "auroc": 0.9677334309895833 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1548, - "fn": 52, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 1490, - "fn": 110, - "accuracy": 0.93125 - } - }, - "auroc": 0.9677334309895833 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1527, - "fn": 73, - "accuracy": 0.954375 - }, - "0.01": { - "tp": 1450, - "fn": 150, - "accuracy": 0.90625 - } - }, - "auroc": 0.9576980305989583 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1527, - "fn": 73, - "accuracy": 0.954375 - }, - "0.01": { - "tp": 1450, - "fn": 150, - "accuracy": 0.90625 - } - }, - "auroc": 0.9576980305989583 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3075, - "fn": 125, - "accuracy": 0.9609375 - }, - "0.01": { - "tp": 2940, - "fn": 260, - "accuracy": 0.91875 - } - }, - "auroc": 0.9627157307942708 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3075, - "fn": 125, - "accuracy": 0.9609375 - }, - "0.01": { - "tp": 2940, - "fn": 260, - "accuracy": 0.91875 - } - }, - "auroc": 0.9627157307942708 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1484, - "fn": 116, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 1346, - "fn": 254, - "accuracy": 0.84125 - } - }, - "auroc": 0.9343791015625 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1484, - "fn": 116, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 1346, - "fn": 254, - "accuracy": 0.84125 - } - }, - "auroc": 0.9343791015625 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1401, - "fn": 199, - "accuracy": 0.875625 - }, - "0.01": { - "tp": 1225, - "fn": 375, - "accuracy": 0.765625 - } - }, - "auroc": 0.9109469889322916 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1401, - "fn": 199, - "accuracy": 0.875625 - }, - "0.01": { - "tp": 1225, - "fn": 375, - "accuracy": 0.765625 - } - }, - "auroc": 0.9109469889322916 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2885, - "fn": 315, - "accuracy": 0.9015625 - }, - "0.01": { - "tp": 2571, - "fn": 629, - "accuracy": 0.8034375 - } - }, - "auroc": 0.9226630452473958 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2885, - "fn": 315, - "accuracy": 0.9015625 - }, - "0.01": { - "tp": 2571, - "fn": 629, - "accuracy": 0.8034375 - } - }, - "auroc": 0.9226630452473958 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9924631184895834 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9924631184895834 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - } - }, - "auroc": 0.990256982421875 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - } - }, - "auroc": 0.990256982421875 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3187, - "fn": 13, - "accuracy": 0.9959375 - } - }, - "auroc": 0.9913600504557292 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3187, - "fn": 13, - "accuracy": 0.9959375 - } - }, - "auroc": 0.9913600504557292 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9935093912760417 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9935093912760417 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1552, - "fn": 48, - "accuracy": 0.97 - } - }, - "auroc": 0.9767893391927083 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1552, - "fn": 48, - "accuracy": 0.97 - } - }, - "auroc": 0.9767893391927083 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3186, - "fn": 14, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 3147, - "fn": 53, - "accuracy": 0.9834375 - } - }, - "auroc": 0.985149365234375 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 3186, - "fn": 14, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 3147, - "fn": 53, - "accuracy": 0.9834375 - } - }, - "auroc": 0.985149365234375 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1521, - "fn": 79, - "accuracy": 0.950625 - }, - "0.01": { - "tp": 1448, - "fn": 152, - "accuracy": 0.905 - } - }, - "auroc": 0.9640538248697916 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1521, - "fn": 79, - "accuracy": 0.950625 - }, - "0.01": { - "tp": 1448, - "fn": 152, - "accuracy": 0.905 - } - }, - "auroc": 0.9640538248697916 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1468, - "fn": 132, - "accuracy": 0.9175 - }, - "0.01": { - "tp": 1371, - "fn": 229, - "accuracy": 0.856875 - } - }, - "auroc": 0.9452570963541667 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 1468, - "fn": 132, - "accuracy": 0.9175 - }, - "0.01": { - "tp": 1371, - "fn": 229, - "accuracy": 0.856875 - } - }, - "auroc": 0.9452570963541667 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2989, - "fn": 211, - "accuracy": 0.9340625 - }, - "0.01": { - "tp": 2819, - "fn": 381, - "accuracy": 0.8809375 - } - }, - "auroc": 0.9546554606119791 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 2989, - "fn": 211, - "accuracy": 0.9340625 - }, - "0.01": { - "tp": 2819, - "fn": 381, - "accuracy": 0.8809375 - } - }, - "auroc": 0.9546554606119791 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 17180, - "fn": 420, - "accuracy": 0.9761363636363637 - }, - "0.01": { - "tp": 16649, - "fn": 951, - "accuracy": 0.945965909090909 - } - }, - "auroc": 0.9762758641098486 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 9547, - "fn": 53, - "accuracy": 0.9944791666666667 - }, - "0.01": { - "tp": 9466, - "fn": 134, - "accuracy": 0.9860416666666667 - } - }, - "auroc": 0.9898300374348958 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 26727, - "fn": 473, - "accuracy": 0.9826102941176471 - }, - "0.01": { - "tp": 26115, - "fn": 1085, - "accuracy": 0.960110294117647 - } - }, - "auroc": 0.981059689989277 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 16690, - "fn": 910, - "accuracy": 0.9482954545454545 - }, - "0.01": { - "tp": 15657, - "fn": 1943, - "accuracy": 0.8896022727272728 - } - }, - "auroc": 0.9560520848129734 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 9364, - "fn": 236, - "accuracy": 0.9754166666666667 - }, - "0.01": { - "tp": 9030, - "fn": 570, - "accuracy": 0.940625 - } - }, - "auroc": 0.9669035454644097 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 26054, - "fn": 1146, - "accuracy": 0.9578676470588235 - }, - "0.01": { - "tp": 24687, - "fn": 2513, - "accuracy": 0.907610294117647 - } - }, - "auroc": 0.9598820121017156 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 33870, - "fn": 1330, - "accuracy": 0.962215909090909 - }, - "0.01": { - "tp": 32306, - "fn": 2894, - "accuracy": 0.9177840909090909 - } - }, - "auroc": 0.966163974461411 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 18911, - "fn": 289, - "accuracy": 0.9849479166666667 - }, - "0.01": { - "tp": 18496, - "fn": 704, - "accuracy": 0.9633333333333334 - } - }, - "auroc": 0.9783667914496528 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "whitespace", - "accuracy": { - "0.05": { - "tp": 52781, - "fn": 1619, - "accuracy": 0.9702389705882353 - }, - "0.01": { - "tp": 50802, - "fn": 3598, - "accuracy": 0.933860294117647 - } - }, - "auroc": 0.9704708510454962 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - } - }, - "auroc": 0.995587158203125 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1598, - "fn": 2, - "accuracy": 0.99875 - } - }, - "auroc": 0.993825341796875 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3197, - "fn": 3, - "accuracy": 0.9990625 - } - }, - "auroc": 0.99470625 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9947333658854166 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9901439290364583 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3191, - "fn": 9, - "accuracy": 0.9971875 - } - }, - "auroc": 0.9924386474609375 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3194, - "fn": 6, - "accuracy": 0.998125 - } - }, - "auroc": 0.9951602620442709 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3194, - "fn": 6, - "accuracy": 0.998125 - } - }, - "auroc": 0.9919846354166667 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 6400, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 6388, - "fn": 12, - "accuracy": 0.998125 - } - }, - "auroc": 0.9935724487304687 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1586, - "fn": 14, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 1557, - "fn": 43, - "accuracy": 0.973125 - } - }, - "auroc": 0.9882420572916666 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - }, - "0.01": { - "tp": 1556, - "fn": 44, - "accuracy": 0.9725 - } - }, - "auroc": 0.9865103841145832 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3162, - "fn": 38, - "accuracy": 0.988125 - }, - "0.01": { - "tp": 3113, - "fn": 87, - "accuracy": 0.9728125 - } - }, - "auroc": 0.9873762207031249 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1581, - "fn": 19, - "accuracy": 0.988125 - }, - "0.01": { - "tp": 1544, - "fn": 56, - "accuracy": 0.965 - } - }, - "auroc": 0.972079296875 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1535, - "fn": 65, - "accuracy": 0.959375 - }, - "0.01": { - "tp": 1487, - "fn": 113, - "accuracy": 0.929375 - } - }, - "auroc": 0.9638255045572917 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3116, - "fn": 84, - "accuracy": 0.97375 - }, - "0.01": { - "tp": 3031, - "fn": 169, - "accuracy": 0.9471875 - } - }, - "auroc": 0.9679524007161457 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3167, - "fn": 33, - "accuracy": 0.9896875 - }, - "0.01": { - "tp": 3101, - "fn": 99, - "accuracy": 0.9690625 - } - }, - "auroc": 0.9801606770833333 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3111, - "fn": 89, - "accuracy": 0.9721875 - }, - "0.01": { - "tp": 3043, - "fn": 157, - "accuracy": 0.9509375 - } - }, - "auroc": 0.9751679443359376 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 6278, - "fn": 122, - "accuracy": 0.9809375 - }, - "0.01": { - "tp": 6144, - "fn": 256, - "accuracy": 0.96 - } - }, - "auroc": 0.9776643107096354 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - } - }, - "auroc": 0.9926324381510417 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 1581, - "fn": 19, - "accuracy": 0.988125 - } - }, - "auroc": 0.9866272135416667 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3184, - "fn": 16, - "accuracy": 0.995 - }, - "0.01": { - "tp": 3170, - "fn": 30, - "accuracy": 0.990625 - } - }, - "auroc": 0.9896298258463542 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - }, - "0.01": { - "tp": 1586, - "fn": 14, - "accuracy": 0.99125 - } - }, - "auroc": 0.990578564453125 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1582, - "fn": 18, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 1532, - "fn": 68, - "accuracy": 0.9575 - } - }, - "auroc": 0.9711081705729165 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3174, - "fn": 26, - "accuracy": 0.991875 - }, - "0.01": { - "tp": 3118, - "fn": 82, - "accuracy": 0.974375 - } - }, - "auroc": 0.9808433675130208 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3186, - "fn": 14, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 3175, - "fn": 25, - "accuracy": 0.9921875 - } - }, - "auroc": 0.9916055013020832 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3172, - "fn": 28, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 3113, - "fn": 87, - "accuracy": 0.9728125 - } - }, - "auroc": 0.9788676920572916 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 6358, - "fn": 42, - "accuracy": 0.9934375 - }, - "0.01": { - "tp": 6288, - "fn": 112, - "accuracy": 0.9825 - } - }, - "auroc": 0.9852365966796874 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1551, - "fn": 49, - "accuracy": 0.969375 - }, - "0.01": { - "tp": 1507, - "fn": 93, - "accuracy": 0.941875 - } - }, - "auroc": 0.9790330078125 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1598, - "fn": 2, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - } - }, - "auroc": 0.9952365885416667 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3149, - "fn": 51, - "accuracy": 0.9840625 - }, - "0.01": { - "tp": 3098, - "fn": 102, - "accuracy": 0.968125 - } - }, - "auroc": 0.9871347981770834 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1580, - "fn": 20, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 1533, - "fn": 67, - "accuracy": 0.958125 - } - }, - "auroc": 0.9742269856770833 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1575, - "fn": 25, - "accuracy": 0.984375 - }, - "0.01": { - "tp": 1544, - "fn": 56, - "accuracy": 0.965 - } - }, - "auroc": 0.97905361328125 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3155, - "fn": 45, - "accuracy": 0.9859375 - }, - "0.01": { - "tp": 3077, - "fn": 123, - "accuracy": 0.9615625 - } - }, - "auroc": 0.9766402994791665 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3131, - "fn": 69, - "accuracy": 0.9784375 - }, - "0.01": { - "tp": 3040, - "fn": 160, - "accuracy": 0.95 - } - }, - "auroc": 0.9766299967447917 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3173, - "fn": 27, - "accuracy": 0.9915625 - }, - "0.01": { - "tp": 3135, - "fn": 65, - "accuracy": 0.9796875 - } - }, - "auroc": 0.9871451009114582 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 6304, - "fn": 96, - "accuracy": 0.985 - }, - "0.01": { - "tp": 6175, - "fn": 225, - "accuracy": 0.96484375 - } - }, - "auroc": 0.981887548828125 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1587, - "fn": 13, - "accuracy": 0.991875 - }, - "0.01": { - "tp": 1553, - "fn": 47, - "accuracy": 0.970625 - } - }, - "auroc": 0.9893400390625 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - } - }, - "auroc": 0.9941066243489582 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3183, - "fn": 17, - "accuracy": 0.9946875 - }, - "0.01": { - "tp": 3145, - "fn": 55, - "accuracy": 0.9828125 - } - }, - "auroc": 0.9917233317057292 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1563, - "fn": 37, - "accuracy": 0.976875 - }, - "0.01": { - "tp": 1480, - "fn": 120, - "accuracy": 0.925 - } - }, - "auroc": 0.956609619140625 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1577, - "fn": 23, - "accuracy": 0.985625 - }, - "0.01": { - "tp": 1511, - "fn": 89, - "accuracy": 0.944375 - } - }, - "auroc": 0.9662674479166666 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3140, - "fn": 60, - "accuracy": 0.98125 - }, - "0.01": { - "tp": 2991, - "fn": 209, - "accuracy": 0.9346875 - } - }, - "auroc": 0.9614385335286458 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3150, - "fn": 50, - "accuracy": 0.984375 - }, - "0.01": { - "tp": 3033, - "fn": 167, - "accuracy": 0.9478125 - } - }, - "auroc": 0.9729748291015625 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3173, - "fn": 27, - "accuracy": 0.9915625 - }, - "0.01": { - "tp": 3103, - "fn": 97, - "accuracy": 0.9696875 - } - }, - "auroc": 0.9801870361328126 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 6323, - "fn": 77, - "accuracy": 0.98796875 - }, - "0.01": { - "tp": 6136, - "fn": 264, - "accuracy": 0.95875 - } - }, - "auroc": 0.9765809326171875 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9945349772135417 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - } - }, - "auroc": 0.993011572265625 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3194, - "fn": 6, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - } - }, - "auroc": 0.9937732747395833 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 1586, - "fn": 14, - "accuracy": 0.99125 - } - }, - "auroc": 0.9912615234375 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1562, - "fn": 38, - "accuracy": 0.97625 - } - }, - "auroc": 0.9816057291666667 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3180, - "fn": 20, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 3148, - "fn": 52, - "accuracy": 0.98375 - } - }, - "auroc": 0.9864336263020834 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3188, - "fn": 12, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 3182, - "fn": 18, - "accuracy": 0.994375 - } - }, - "auroc": 0.9928982503255208 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3186, - "fn": 14, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 3156, - "fn": 44, - "accuracy": 0.98625 - } - }, - "auroc": 0.9873086507161457 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 6374, - "fn": 26, - "accuracy": 0.9959375 - }, - "0.01": { - "tp": 6338, - "fn": 62, - "accuracy": 0.9903125 - } - }, - "auroc": 0.9901034505208333 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1557, - "fn": 43, - "accuracy": 0.973125 - }, - "0.01": { - "tp": 1526, - "fn": 74, - "accuracy": 0.95375 - } - }, - "auroc": 0.9747910970052083 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1557, - "fn": 43, - "accuracy": 0.973125 - }, - "0.01": { - "tp": 1526, - "fn": 74, - "accuracy": 0.95375 - } - }, - "auroc": 0.9747910970052083 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1554, - "fn": 46, - "accuracy": 0.97125 - }, - "0.01": { - "tp": 1505, - "fn": 95, - "accuracy": 0.940625 - } - }, - "auroc": 0.9674541015625 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1554, - "fn": 46, - "accuracy": 0.97125 - }, - "0.01": { - "tp": 1505, - "fn": 95, - "accuracy": 0.940625 - } - }, - "auroc": 0.9674541015625 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3111, - "fn": 89, - "accuracy": 0.9721875 - }, - "0.01": { - "tp": 3031, - "fn": 169, - "accuracy": 0.9471875 - } - }, - "auroc": 0.9711225992838541 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3111, - "fn": 89, - "accuracy": 0.9721875 - }, - "0.01": { - "tp": 3031, - "fn": 169, - "accuracy": 0.9471875 - } - }, - "auroc": 0.9711225992838541 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1563, - "fn": 37, - "accuracy": 0.976875 - }, - "0.01": { - "tp": 1500, - "fn": 100, - "accuracy": 0.9375 - } - }, - "auroc": 0.9635181315104167 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1563, - "fn": 37, - "accuracy": 0.976875 - }, - "0.01": { - "tp": 1500, - "fn": 100, - "accuracy": 0.9375 - } - }, - "auroc": 0.9635181315104167 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1547, - "fn": 53, - "accuracy": 0.966875 - }, - "0.01": { - "tp": 1433, - "fn": 167, - "accuracy": 0.895625 - } - }, - "auroc": 0.9485077962239585 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1547, - "fn": 53, - "accuracy": 0.966875 - }, - "0.01": { - "tp": 1433, - "fn": 167, - "accuracy": 0.895625 - } - }, - "auroc": 0.9485077962239585 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3110, - "fn": 90, - "accuracy": 0.971875 - }, - "0.01": { - "tp": 2933, - "fn": 267, - "accuracy": 0.9165625 - } - }, - "auroc": 0.9560129638671876 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3110, - "fn": 90, - "accuracy": 0.971875 - }, - "0.01": { - "tp": 2933, - "fn": 267, - "accuracy": 0.9165625 - } - }, - "auroc": 0.9560129638671876 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9911006510416667 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9911006510416667 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - } - }, - "auroc": 0.9889077962239583 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - } - }, - "auroc": 0.9889077962239583 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3189, - "fn": 11, - "accuracy": 0.9965625 - } - }, - "auroc": 0.9900042236328126 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3189, - "fn": 11, - "accuracy": 0.9965625 - } - }, - "auroc": 0.9900042236328126 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.993077587890625 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.993077587890625 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1598, - "fn": 2, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 1580, - "fn": 20, - "accuracy": 0.9875 - } - }, - "auroc": 0.9809000325520834 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1598, - "fn": 2, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 1580, - "fn": 20, - "accuracy": 0.9875 - } - }, - "auroc": 0.9809000325520834 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3197, - "fn": 3, - "accuracy": 0.9990625 - }, - "0.01": { - "tp": 3176, - "fn": 24, - "accuracy": 0.9925 - } - }, - "auroc": 0.9869888102213542 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3197, - "fn": 3, - "accuracy": 0.9990625 - }, - "0.01": { - "tp": 3176, - "fn": 24, - "accuracy": 0.9925 - } - }, - "auroc": 0.9869888102213542 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1569, - "fn": 31, - "accuracy": 0.980625 - }, - "0.01": { - "tp": 1526, - "fn": 74, - "accuracy": 0.95375 - } - }, - "auroc": 0.9776026041666667 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1569, - "fn": 31, - "accuracy": 0.980625 - }, - "0.01": { - "tp": 1526, - "fn": 74, - "accuracy": 0.95375 - } - }, - "auroc": 0.9776026041666667 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1544, - "fn": 56, - "accuracy": 0.965 - }, - "0.01": { - "tp": 1453, - "fn": 147, - "accuracy": 0.908125 - } - }, - "auroc": 0.9629502766927084 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 1544, - "fn": 56, - "accuracy": 0.965 - }, - "0.01": { - "tp": 1453, - "fn": 147, - "accuracy": 0.908125 - } - }, - "auroc": 0.9629502766927084 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3113, - "fn": 87, - "accuracy": 0.9728125 - }, - "0.01": { - "tp": 2979, - "fn": 221, - "accuracy": 0.9309375 - } - }, - "auroc": 0.9702764404296875 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 3113, - "fn": 87, - "accuracy": 0.9728125 - }, - "0.01": { - "tp": 2979, - "fn": 221, - "accuracy": 0.9309375 - } - }, - "auroc": 0.9702764404296875 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 17399, - "fn": 201, - "accuracy": 0.9885795454545454 - }, - "0.01": { - "tp": 17144, - "fn": 456, - "accuracy": 0.9740909090909091 - } - }, - "auroc": 0.9854054317589962 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 9557, - "fn": 43, - "accuracy": 0.9955208333333333 - }, - "0.01": { - "tp": 9512, - "fn": 88, - "accuracy": 0.9908333333333333 - } - }, - "auroc": 0.9915529541015624 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 26956, - "fn": 244, - "accuracy": 0.9910294117647059 - }, - "0.01": { - "tp": 26656, - "fn": 544, - "accuracy": 0.98 - } - }, - "auroc": 0.9875751455269608 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 17344, - "fn": 256, - "accuracy": 0.9854545454545455 - }, - "0.01": { - "tp": 16889, - "fn": 711, - "accuracy": 0.9596022727272727 - } - }, - "auroc": 0.9752917598839962 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 9458, - "fn": 142, - "accuracy": 0.9852083333333334 - }, - "0.01": { - "tp": 9232, - "fn": 368, - "accuracy": 0.9616666666666667 - } - }, - "auroc": 0.9753340657552083 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 26802, - "fn": 398, - "accuracy": 0.9853676470588235 - }, - "0.01": { - "tp": 26121, - "fn": 1079, - "accuracy": 0.9603308823529412 - } - }, - "auroc": 0.9753066913679534 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 34743, - "fn": 457, - "accuracy": 0.9870170454545455 - }, - "0.01": { - "tp": 34033, - "fn": 1167, - "accuracy": 0.9668465909090909 - } - }, - "auroc": 0.9803485958214962 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 19015, - "fn": 185, - "accuracy": 0.9903645833333333 - }, - "0.01": { - "tp": 18744, - "fn": 456, - "accuracy": 0.97625 - } - }, - "auroc": 0.9834435099283855 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "upper_lower", - "accuracy": { - "0.05": { - "tp": 53758, - "fn": 642, - "accuracy": 0.9881985294117647 - }, - "0.01": { - "tp": 52777, - "fn": 1623, - "accuracy": 0.9701654411764706 - } - }, - "auroc": 0.9814409184474571 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - } - }, - "auroc": 0.9951512532552083 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - } - }, - "auroc": 0.9920146158854167 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3186, - "fn": 14, - "accuracy": 0.995625 - } - }, - "auroc": 0.9935829345703124 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9941670572916668 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - }, - "0.01": { - "tp": 1560, - "fn": 40, - "accuracy": 0.975 - } - }, - "auroc": 0.9833861653645833 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3191, - "fn": 9, - "accuracy": 0.9971875 - }, - "0.01": { - "tp": 3156, - "fn": 44, - "accuracy": 0.98625 - } - }, - "auroc": 0.988776611328125 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3199, - "fn": 1, - "accuracy": 0.9996875 - }, - "0.01": { - "tp": 3193, - "fn": 7, - "accuracy": 0.9978125 - } - }, - "auroc": 0.9946591552734375 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3192, - "fn": 8, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 3149, - "fn": 51, - "accuracy": 0.9840625 - } - }, - "auroc": 0.9877003906249999 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 6391, - "fn": 9, - "accuracy": 0.99859375 - }, - "0.01": { - "tp": 6342, - "fn": 58, - "accuracy": 0.9909375 - } - }, - "auroc": 0.9911797729492187 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1560, - "fn": 40, - "accuracy": 0.975 - }, - "0.01": { - "tp": 1488, - "fn": 112, - "accuracy": 0.93 - } - }, - "auroc": 0.9713520182291668 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1571, - "fn": 29, - "accuracy": 0.981875 - }, - "0.01": { - "tp": 1542, - "fn": 58, - "accuracy": 0.96375 - } - }, - "auroc": 0.983770556640625 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3131, - "fn": 69, - "accuracy": 0.9784375 - }, - "0.01": { - "tp": 3030, - "fn": 170, - "accuracy": 0.946875 - } - }, - "auroc": 0.9775612874348958 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1443, - "fn": 157, - "accuracy": 0.901875 - }, - "0.01": { - "tp": 1277, - "fn": 323, - "accuracy": 0.798125 - } - }, - "auroc": 0.9253209309895832 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1527, - "fn": 73, - "accuracy": 0.954375 - }, - "0.01": { - "tp": 1473, - "fn": 127, - "accuracy": 0.920625 - } - }, - "auroc": 0.9615456380208334 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2970, - "fn": 230, - "accuracy": 0.928125 - }, - "0.01": { - "tp": 2750, - "fn": 450, - "accuracy": 0.859375 - } - }, - "auroc": 0.9434332845052082 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3003, - "fn": 197, - "accuracy": 0.9384375 - }, - "0.01": { - "tp": 2765, - "fn": 435, - "accuracy": 0.8640625 - } - }, - "auroc": 0.948336474609375 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3098, - "fn": 102, - "accuracy": 0.968125 - }, - "0.01": { - "tp": 3015, - "fn": 185, - "accuracy": 0.9421875 - } - }, - "auroc": 0.9726580973307292 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 6101, - "fn": 299, - "accuracy": 0.95328125 - }, - "0.01": { - "tp": 5780, - "fn": 620, - "accuracy": 0.903125 - } - }, - "auroc": 0.9604972859700521 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1580, - "fn": 20, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 1553, - "fn": 47, - "accuracy": 0.970625 - } - }, - "auroc": 0.9849125000000001 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1582, - "fn": 18, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 1537, - "fn": 63, - "accuracy": 0.960625 - } - }, - "auroc": 0.9740000651041666 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3162, - "fn": 38, - "accuracy": 0.988125 - }, - "0.01": { - "tp": 3090, - "fn": 110, - "accuracy": 0.965625 - } - }, - "auroc": 0.9794562825520834 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1572, - "fn": 28, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 1530, - "fn": 70, - "accuracy": 0.95625 - } - }, - "auroc": 0.9803068522135416 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1565, - "fn": 35, - "accuracy": 0.978125 - }, - "0.01": { - "tp": 1462, - "fn": 138, - "accuracy": 0.91375 - } - }, - "auroc": 0.956943359375 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3137, - "fn": 63, - "accuracy": 0.9803125 - }, - "0.01": { - "tp": 2992, - "fn": 208, - "accuracy": 0.935 - } - }, - "auroc": 0.9686251057942709 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3152, - "fn": 48, - "accuracy": 0.985 - }, - "0.01": { - "tp": 3083, - "fn": 117, - "accuracy": 0.9634375 - } - }, - "auroc": 0.9826096761067709 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3147, - "fn": 53, - "accuracy": 0.9834375 - }, - "0.01": { - "tp": 2999, - "fn": 201, - "accuracy": 0.9371875 - } - }, - "auroc": 0.9654717122395833 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 6299, - "fn": 101, - "accuracy": 0.98421875 - }, - "0.01": { - "tp": 6082, - "fn": 318, - "accuracy": 0.9503125 - } - }, - "auroc": 0.9740406941731771 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1518, - "fn": 82, - "accuracy": 0.94875 - }, - "0.01": { - "tp": 1396, - "fn": 204, - "accuracy": 0.8725 - } - }, - "auroc": 0.9518605631510417 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 1565, - "fn": 35, - "accuracy": 0.978125 - } - }, - "auroc": 0.9861591634114584 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3113, - "fn": 87, - "accuracy": 0.9728125 - }, - "0.01": { - "tp": 2961, - "fn": 239, - "accuracy": 0.9253125 - } - }, - "auroc": 0.96900986328125 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1481, - "fn": 119, - "accuracy": 0.925625 - }, - "0.01": { - "tp": 1222, - "fn": 378, - "accuracy": 0.76375 - } - }, - "auroc": 0.9211118815104167 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1543, - "fn": 57, - "accuracy": 0.964375 - }, - "0.01": { - "tp": 1448, - "fn": 152, - "accuracy": 0.905 - } - }, - "auroc": 0.9501723307291667 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3024, - "fn": 176, - "accuracy": 0.945 - }, - "0.01": { - "tp": 2670, - "fn": 530, - "accuracy": 0.834375 - } - }, - "auroc": 0.9356421061197916 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2999, - "fn": 201, - "accuracy": 0.9371875 - }, - "0.01": { - "tp": 2618, - "fn": 582, - "accuracy": 0.818125 - } - }, - "auroc": 0.9364862223307291 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3138, - "fn": 62, - "accuracy": 0.980625 - }, - "0.01": { - "tp": 3013, - "fn": 187, - "accuracy": 0.9415625 - } - }, - "auroc": 0.9681657470703126 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 6137, - "fn": 263, - "accuracy": 0.95890625 - }, - "0.01": { - "tp": 5631, - "fn": 769, - "accuracy": 0.87984375 - } - }, - "auroc": 0.9523259847005209 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1563, - "fn": 37, - "accuracy": 0.976875 - }, - "0.01": { - "tp": 1501, - "fn": 99, - "accuracy": 0.938125 - } - }, - "auroc": 0.9778763997395834 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1587, - "fn": 13, - "accuracy": 0.991875 - }, - "0.01": { - "tp": 1553, - "fn": 47, - "accuracy": 0.970625 - } - }, - "auroc": 0.97998427734375 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3150, - "fn": 50, - "accuracy": 0.984375 - }, - "0.01": { - "tp": 3054, - "fn": 146, - "accuracy": 0.954375 - } - }, - "auroc": 0.9789303385416667 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1401, - "fn": 199, - "accuracy": 0.875625 - }, - "0.01": { - "tp": 1106, - "fn": 494, - "accuracy": 0.69125 - } - }, - "auroc": 0.8974861165364583 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1548, - "fn": 52, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 1449, - "fn": 151, - "accuracy": 0.905625 - } - }, - "auroc": 0.9504881022135416 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2949, - "fn": 251, - "accuracy": 0.9215625 - }, - "0.01": { - "tp": 2555, - "fn": 645, - "accuracy": 0.7984375 - } - }, - "auroc": 0.9239871093750001 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2964, - "fn": 236, - "accuracy": 0.92625 - }, - "0.01": { - "tp": 2607, - "fn": 593, - "accuracy": 0.8146875 - } - }, - "auroc": 0.9376812581380208 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3135, - "fn": 65, - "accuracy": 0.9796875 - }, - "0.01": { - "tp": 3002, - "fn": 198, - "accuracy": 0.938125 - } - }, - "auroc": 0.9652361897786459 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 6099, - "fn": 301, - "accuracy": 0.95296875 - }, - "0.01": { - "tp": 5609, - "fn": 791, - "accuracy": 0.87640625 - } - }, - "auroc": 0.9514587239583333 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 1580, - "fn": 20, - "accuracy": 0.9875 - } - }, - "auroc": 0.9909703125 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - } - }, - "auroc": 0.988282763671875 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3180, - "fn": 20, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 3156, - "fn": 44, - "accuracy": 0.98625 - } - }, - "auroc": 0.9896265380859375 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1581, - "fn": 19, - "accuracy": 0.988125 - }, - "0.01": { - "tp": 1551, - "fn": 49, - "accuracy": 0.969375 - } - }, - "auroc": 0.98466884765625 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1554, - "fn": 46, - "accuracy": 0.97125 - }, - "0.01": { - "tp": 1475, - "fn": 125, - "accuracy": 0.921875 - } - }, - "auroc": 0.9642508300781251 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3135, - "fn": 65, - "accuracy": 0.9796875 - }, - "0.01": { - "tp": 3026, - "fn": 174, - "accuracy": 0.945625 - } - }, - "auroc": 0.9744598388671875 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3172, - "fn": 28, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 3131, - "fn": 69, - "accuracy": 0.9784375 - } - }, - "auroc": 0.987819580078125 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3143, - "fn": 57, - "accuracy": 0.9821875 - }, - "0.01": { - "tp": 3051, - "fn": 149, - "accuracy": 0.9534375 - } - }, - "auroc": 0.9762667968750001 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 6315, - "fn": 85, - "accuracy": 0.98671875 - }, - "0.01": { - "tp": 6182, - "fn": 218, - "accuracy": 0.9659375 - } - }, - "auroc": 0.9820431884765626 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1521, - "fn": 79, - "accuracy": 0.950625 - }, - "0.01": { - "tp": 1400, - "fn": 200, - "accuracy": 0.875 - } - }, - "auroc": 0.95185732421875 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1521, - "fn": 79, - "accuracy": 0.950625 - }, - "0.01": { - "tp": 1400, - "fn": 200, - "accuracy": 0.875 - } - }, - "auroc": 0.95185732421875 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1480, - "fn": 120, - "accuracy": 0.925 - }, - "0.01": { - "tp": 1356, - "fn": 244, - "accuracy": 0.8475 - } - }, - "auroc": 0.9381492838541666 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1480, - "fn": 120, - "accuracy": 0.925 - }, - "0.01": { - "tp": 1356, - "fn": 244, - "accuracy": 0.8475 - } - }, - "auroc": 0.9381492838541666 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3001, - "fn": 199, - "accuracy": 0.9378125 - }, - "0.01": { - "tp": 2756, - "fn": 444, - "accuracy": 0.86125 - } - }, - "auroc": 0.9450033040364583 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3001, - "fn": 199, - "accuracy": 0.9378125 - }, - "0.01": { - "tp": 2756, - "fn": 444, - "accuracy": 0.86125 - } - }, - "auroc": 0.9450033040364583 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1415, - "fn": 185, - "accuracy": 0.884375 - }, - "0.01": { - "tp": 1233, - "fn": 367, - "accuracy": 0.770625 - } - }, - "auroc": 0.9153590332031251 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1415, - "fn": 185, - "accuracy": 0.884375 - }, - "0.01": { - "tp": 1233, - "fn": 367, - "accuracy": 0.770625 - } - }, - "auroc": 0.9153590332031251 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1348, - "fn": 252, - "accuracy": 0.8425 - }, - "0.01": { - "tp": 1125, - "fn": 475, - "accuracy": 0.703125 - } - }, - "auroc": 0.8926791992187499 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1348, - "fn": 252, - "accuracy": 0.8425 - }, - "0.01": { - "tp": 1125, - "fn": 475, - "accuracy": 0.703125 - } - }, - "auroc": 0.8926791992187499 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2763, - "fn": 437, - "accuracy": 0.8634375 - }, - "0.01": { - "tp": 2358, - "fn": 842, - "accuracy": 0.736875 - } - }, - "auroc": 0.9040191162109374 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2763, - "fn": 437, - "accuracy": 0.8634375 - }, - "0.01": { - "tp": 2358, - "fn": 842, - "accuracy": 0.736875 - } - }, - "auroc": 0.9040191162109374 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - } - }, - "auroc": 0.9879233398437501 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - } - }, - "auroc": 0.9879233398437501 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1586, - "fn": 14, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 1571, - "fn": 29, - "accuracy": 0.981875 - } - }, - "auroc": 0.9843487955729167 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1586, - "fn": 14, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 1571, - "fn": 29, - "accuracy": 0.981875 - } - }, - "auroc": 0.9843487955729167 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3180, - "fn": 20, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 3155, - "fn": 45, - "accuracy": 0.9859375 - } - }, - "auroc": 0.9861360677083334 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3180, - "fn": 20, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 3155, - "fn": 45, - "accuracy": 0.9859375 - } - }, - "auroc": 0.9861360677083334 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - } - }, - "auroc": 0.9910857910156251 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - } - }, - "auroc": 0.9910857910156251 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1574, - "fn": 26, - "accuracy": 0.98375 - }, - "0.01": { - "tp": 1491, - "fn": 109, - "accuracy": 0.931875 - } - }, - "auroc": 0.9651382161458333 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1574, - "fn": 26, - "accuracy": 0.98375 - }, - "0.01": { - "tp": 1491, - "fn": 109, - "accuracy": 0.931875 - } - }, - "auroc": 0.9651382161458333 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3168, - "fn": 32, - "accuracy": 0.99 - }, - "0.01": { - "tp": 3083, - "fn": 117, - "accuracy": 0.9634375 - } - }, - "auroc": 0.9781120035807293 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 3168, - "fn": 32, - "accuracy": 0.99 - }, - "0.01": { - "tp": 3083, - "fn": 117, - "accuracy": 0.9634375 - } - }, - "auroc": 0.9781120035807293 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1479, - "fn": 121, - "accuracy": 0.924375 - }, - "0.01": { - "tp": 1376, - "fn": 224, - "accuracy": 0.86 - } - }, - "auroc": 0.9482429524739584 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1479, - "fn": 121, - "accuracy": 0.924375 - }, - "0.01": { - "tp": 1376, - "fn": 224, - "accuracy": 0.86 - } - }, - "auroc": 0.9482429524739584 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1418, - "fn": 182, - "accuracy": 0.88625 - }, - "0.01": { - "tp": 1276, - "fn": 324, - "accuracy": 0.7975 - } - }, - "auroc": 0.9300056803385416 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 1418, - "fn": 182, - "accuracy": 0.88625 - }, - "0.01": { - "tp": 1276, - "fn": 324, - "accuracy": 0.7975 - } - }, - "auroc": 0.9300056803385416 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2897, - "fn": 303, - "accuracy": 0.9053125 - }, - "0.01": { - "tp": 2652, - "fn": 548, - "accuracy": 0.82875 - } - }, - "auroc": 0.9391243164062502 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 2897, - "fn": 303, - "accuracy": 0.9053125 - }, - "0.01": { - "tp": 2652, - "fn": 548, - "accuracy": 0.82875 - } - }, - "auroc": 0.9391243164062502 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 17015, - "fn": 585, - "accuracy": 0.9667613636363637 - }, - "0.01": { - "tp": 16300, - "fn": 1300, - "accuracy": 0.9261363636363636 - } - }, - "auroc": 0.9696901352391099 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 9524, - "fn": 76, - "accuracy": 0.9920833333333333 - }, - "0.01": { - "tp": 9362, - "fn": 238, - "accuracy": 0.9752083333333333 - } - }, - "auroc": 0.984035240342882 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 26539, - "fn": 661, - "accuracy": 0.9756985294117647 - }, - "0.01": { - "tp": 25662, - "fn": 1538, - "accuracy": 0.9434558823529412 - } - }, - "auroc": 0.9747531135110294 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 16483, - "fn": 1117, - "accuracy": 0.9365340909090909 - }, - "0.01": { - "tp": 15101, - "fn": 2499, - "accuracy": 0.8580113636363637 - } - }, - "auroc": 0.9466711692116476 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 9329, - "fn": 271, - "accuracy": 0.9717708333333334 - }, - "0.01": { - "tp": 8867, - "fn": 733, - "accuracy": 0.9236458333333334 - } - }, - "auroc": 0.9611310709635417 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 25812, - "fn": 1388, - "accuracy": 0.9489705882352941 - }, - "0.01": { - "tp": 23968, - "fn": 3232, - "accuracy": 0.8811764705882353 - } - }, - "auroc": 0.9517746639476103 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 33498, - "fn": 1702, - "accuracy": 0.9516477272727273 - }, - "0.01": { - "tp": 31401, - "fn": 3799, - "accuracy": 0.8920738636363637 - } - }, - "auroc": 0.9581806522253786 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 18853, - "fn": 347, - "accuracy": 0.9819270833333333 - }, - "0.01": { - "tp": 18229, - "fn": 971, - "accuracy": 0.9494270833333334 - } - }, - "auroc": 0.9725831556532117 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "synonym", - "accuracy": { - "0.05": { - "tp": 52351, - "fn": 2049, - "accuracy": 0.9623345588235294 - }, - "0.01": { - "tp": 49630, - "fn": 4770, - "accuracy": 0.9123161764705883 - } - }, - "auroc": 0.9632638887293199 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - } - }, - "auroc": 0.9966502766927083 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1598, - "fn": 2, - "accuracy": 0.99875 - } - }, - "auroc": 0.99512236328125 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3197, - "fn": 3, - "accuracy": 0.9990625 - } - }, - "auroc": 0.9958863199869792 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1598, - "fn": 2, - "accuracy": 0.99875 - } - }, - "auroc": 0.9958223795572917 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - } - }, - "auroc": 0.9915072591145834 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3199, - "fn": 1, - "accuracy": 0.9996875 - }, - "0.01": { - "tp": 3187, - "fn": 13, - "accuracy": 0.9959375 - } - }, - "auroc": 0.9936648193359375 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3197, - "fn": 3, - "accuracy": 0.9990625 - } - }, - "auroc": 0.9962363281250001 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3199, - "fn": 1, - "accuracy": 0.9996875 - }, - "0.01": { - "tp": 3187, - "fn": 13, - "accuracy": 0.9959375 - } - }, - "auroc": 0.9933148111979166 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 6399, - "fn": 1, - "accuracy": 0.99984375 - }, - "0.01": { - "tp": 6384, - "fn": 16, - "accuracy": 0.9975 - } - }, - "auroc": 0.9947755696614583 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1563, - "fn": 37, - "accuracy": 0.976875 - }, - "0.01": { - "tp": 1491, - "fn": 109, - "accuracy": 0.931875 - } - }, - "auroc": 0.9724764322916668 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - }, - "0.01": { - "tp": 1553, - "fn": 47, - "accuracy": 0.970625 - } - }, - "auroc": 0.9868575032552083 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3139, - "fn": 61, - "accuracy": 0.9809375 - }, - "0.01": { - "tp": 3044, - "fn": 156, - "accuracy": 0.95125 - } - }, - "auroc": 0.9796669677734374 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1524, - "fn": 76, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 1424, - "fn": 176, - "accuracy": 0.89 - } - }, - "auroc": 0.9523849609375 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1537, - "fn": 63, - "accuracy": 0.960625 - }, - "0.01": { - "tp": 1484, - "fn": 116, - "accuracy": 0.9275 - } - }, - "auroc": 0.9654538899739584 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3061, - "fn": 139, - "accuracy": 0.9565625 - }, - "0.01": { - "tp": 2908, - "fn": 292, - "accuracy": 0.90875 - } - }, - "auroc": 0.9589194254557291 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3087, - "fn": 113, - "accuracy": 0.9646875 - }, - "0.01": { - "tp": 2915, - "fn": 285, - "accuracy": 0.9109375 - } - }, - "auroc": 0.9624306966145834 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3113, - "fn": 87, - "accuracy": 0.9728125 - }, - "0.01": { - "tp": 3037, - "fn": 163, - "accuracy": 0.9490625 - } - }, - "auroc": 0.9761556966145833 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 6200, - "fn": 200, - "accuracy": 0.96875 - }, - "0.01": { - "tp": 5952, - "fn": 448, - "accuracy": 0.93 - } - }, - "auroc": 0.9692931966145832 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 1585, - "fn": 15, - "accuracy": 0.990625 - } - }, - "auroc": 0.9929725260416666 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 1578, - "fn": 22, - "accuracy": 0.98625 - } - }, - "auroc": 0.98678017578125 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3181, - "fn": 19, - "accuracy": 0.9940625 - }, - "0.01": { - "tp": 3163, - "fn": 37, - "accuracy": 0.9884375 - } - }, - "auroc": 0.9898763509114583 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1588, - "fn": 12, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 1578, - "fn": 22, - "accuracy": 0.98625 - } - }, - "auroc": 0.990692138671875 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1582, - "fn": 18, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 1527, - "fn": 73, - "accuracy": 0.954375 - } - }, - "auroc": 0.9704941080729167 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3170, - "fn": 30, - "accuracy": 0.990625 - }, - "0.01": { - "tp": 3105, - "fn": 95, - "accuracy": 0.9703125 - } - }, - "auroc": 0.9805931233723958 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3179, - "fn": 21, - "accuracy": 0.9934375 - }, - "0.01": { - "tp": 3163, - "fn": 37, - "accuracy": 0.9884375 - } - }, - "auroc": 0.9918323323567708 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3172, - "fn": 28, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 3105, - "fn": 95, - "accuracy": 0.9703125 - } - }, - "auroc": 0.9786371419270833 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 6351, - "fn": 49, - "accuracy": 0.99234375 - }, - "0.01": { - "tp": 6268, - "fn": 132, - "accuracy": 0.979375 - } - }, - "auroc": 0.9852347371419271 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1518, - "fn": 82, - "accuracy": 0.94875 - }, - "0.01": { - "tp": 1383, - "fn": 217, - "accuracy": 0.864375 - } - }, - "auroc": 0.9516840494791666 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1577, - "fn": 23, - "accuracy": 0.985625 - } - }, - "auroc": 0.9936931477864583 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3114, - "fn": 86, - "accuracy": 0.973125 - }, - "0.01": { - "tp": 2960, - "fn": 240, - "accuracy": 0.925 - } - }, - "auroc": 0.9726885986328124 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1546, - "fn": 54, - "accuracy": 0.96625 - }, - "0.01": { - "tp": 1423, - "fn": 177, - "accuracy": 0.889375 - } - }, - "auroc": 0.9563634440104167 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1558, - "fn": 42, - "accuracy": 0.97375 - }, - "0.01": { - "tp": 1511, - "fn": 89, - "accuracy": 0.944375 - } - }, - "auroc": 0.9662256022135417 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3104, - "fn": 96, - "accuracy": 0.97 - }, - "0.01": { - "tp": 2934, - "fn": 266, - "accuracy": 0.916875 - } - }, - "auroc": 0.9612945231119792 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3064, - "fn": 136, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 2806, - "fn": 394, - "accuracy": 0.876875 - } - }, - "auroc": 0.9540237467447916 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3154, - "fn": 46, - "accuracy": 0.985625 - }, - "0.01": { - "tp": 3088, - "fn": 112, - "accuracy": 0.965 - } - }, - "auroc": 0.979959375 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 6218, - "fn": 182, - "accuracy": 0.9715625 - }, - "0.01": { - "tp": 5894, - "fn": 506, - "accuracy": 0.9209375 - } - }, - "auroc": 0.9669915608723958 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1565, - "fn": 35, - "accuracy": 0.978125 - }, - "0.01": { - "tp": 1502, - "fn": 98, - "accuracy": 0.93875 - } - }, - "auroc": 0.9786271158854167 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 1580, - "fn": 20, - "accuracy": 0.9875 - } - }, - "auroc": 0.9893461263020833 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3156, - "fn": 44, - "accuracy": 0.98625 - }, - "0.01": { - "tp": 3082, - "fn": 118, - "accuracy": 0.963125 - } - }, - "auroc": 0.9839866210937499 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1465, - "fn": 135, - "accuracy": 0.915625 - }, - "0.01": { - "tp": 1293, - "fn": 307, - "accuracy": 0.808125 - } - }, - "auroc": 0.9275569173177083 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1544, - "fn": 56, - "accuracy": 0.965 - }, - "0.01": { - "tp": 1460, - "fn": 140, - "accuracy": 0.9125 - } - }, - "auroc": 0.9563819173177083 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3009, - "fn": 191, - "accuracy": 0.9403125 - }, - "0.01": { - "tp": 2753, - "fn": 447, - "accuracy": 0.8603125 - } - }, - "auroc": 0.9419694173177084 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3030, - "fn": 170, - "accuracy": 0.946875 - }, - "0.01": { - "tp": 2795, - "fn": 405, - "accuracy": 0.8734375 - } - }, - "auroc": 0.9530920166015625 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3135, - "fn": 65, - "accuracy": 0.9796875 - }, - "0.01": { - "tp": 3040, - "fn": 160, - "accuracy": 0.95 - } - }, - "auroc": 0.9728640218098958 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 6165, - "fn": 235, - "accuracy": 0.96328125 - }, - "0.01": { - "tp": 5835, - "fn": 565, - "accuracy": 0.91171875 - } - }, - "auroc": 0.9629780192057291 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - } - }, - "auroc": 0.9947170572916666 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - } - }, - "auroc": 0.9940895182291667 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3192, - "fn": 8, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 3185, - "fn": 15, - "accuracy": 0.9953125 - } - }, - "auroc": 0.9944032877604166 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1588, - "fn": 12, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 1583, - "fn": 17, - "accuracy": 0.989375 - } - }, - "auroc": 0.9913267252604165 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1570, - "fn": 30, - "accuracy": 0.98125 - }, - "0.01": { - "tp": 1534, - "fn": 66, - "accuracy": 0.95875 - } - }, - "auroc": 0.9779560384114585 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3158, - "fn": 42, - "accuracy": 0.986875 - }, - "0.01": { - "tp": 3117, - "fn": 83, - "accuracy": 0.9740625 - } - }, - "auroc": 0.9846413818359374 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3184, - "fn": 16, - "accuracy": 0.995 - }, - "0.01": { - "tp": 3177, - "fn": 23, - "accuracy": 0.9928125 - } - }, - "auroc": 0.9930218912760416 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3166, - "fn": 34, - "accuracy": 0.989375 - }, - "0.01": { - "tp": 3125, - "fn": 75, - "accuracy": 0.9765625 - } - }, - "auroc": 0.9860227783203125 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 6350, - "fn": 50, - "accuracy": 0.9921875 - }, - "0.01": { - "tp": 6302, - "fn": 98, - "accuracy": 0.9846875 - } - }, - "auroc": 0.9895223347981772 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1546, - "fn": 54, - "accuracy": 0.96625 - }, - "0.01": { - "tp": 1490, - "fn": 110, - "accuracy": 0.93125 - } - }, - "auroc": 0.9684078287760416 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1546, - "fn": 54, - "accuracy": 0.96625 - }, - "0.01": { - "tp": 1490, - "fn": 110, - "accuracy": 0.93125 - } - }, - "auroc": 0.9684078287760416 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1524, - "fn": 76, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 1450, - "fn": 150, - "accuracy": 0.90625 - } - }, - "auroc": 0.9585711100260417 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1524, - "fn": 76, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 1450, - "fn": 150, - "accuracy": 0.90625 - } - }, - "auroc": 0.9585711100260417 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3070, - "fn": 130, - "accuracy": 0.959375 - }, - "0.01": { - "tp": 2940, - "fn": 260, - "accuracy": 0.91875 - } - }, - "auroc": 0.9634894694010416 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3070, - "fn": 130, - "accuracy": 0.959375 - }, - "0.01": { - "tp": 2940, - "fn": 260, - "accuracy": 0.91875 - } - }, - "auroc": 0.9634894694010416 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1509, - "fn": 91, - "accuracy": 0.943125 - }, - "0.01": { - "tp": 1422, - "fn": 178, - "accuracy": 0.88875 - } - }, - "auroc": 0.9478448404947917 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1509, - "fn": 91, - "accuracy": 0.943125 - }, - "0.01": { - "tp": 1422, - "fn": 178, - "accuracy": 0.88875 - } - }, - "auroc": 0.9478448404947917 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1453, - "fn": 147, - "accuracy": 0.908125 - }, - "0.01": { - "tp": 1320, - "fn": 280, - "accuracy": 0.825 - } - }, - "auroc": 0.9295477213541666 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1453, - "fn": 147, - "accuracy": 0.908125 - }, - "0.01": { - "tp": 1320, - "fn": 280, - "accuracy": 0.825 - } - }, - "auroc": 0.9295477213541666 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2962, - "fn": 238, - "accuracy": 0.925625 - }, - "0.01": { - "tp": 2742, - "fn": 458, - "accuracy": 0.856875 - } - }, - "auroc": 0.9386962809244791 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 2962, - "fn": 238, - "accuracy": 0.925625 - }, - "0.01": { - "tp": 2742, - "fn": 458, - "accuracy": 0.856875 - } - }, - "auroc": 0.9386962809244791 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9924008789062501 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9924008789062501 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - } - }, - "auroc": 0.9902497721354168 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - } - }, - "auroc": 0.9902497721354168 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3185, - "fn": 15, - "accuracy": 0.9953125 - } - }, - "auroc": 0.9913253255208334 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3185, - "fn": 15, - "accuracy": 0.9953125 - } - }, - "auroc": 0.9913253255208334 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9937511393229167 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9937511393229167 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1593, - "fn": 7, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 1568, - "fn": 32, - "accuracy": 0.98 - } - }, - "auroc": 0.9810590006510417 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1593, - "fn": 7, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 1568, - "fn": 32, - "accuracy": 0.98 - } - }, - "auroc": 0.9810590006510417 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3163, - "fn": 37, - "accuracy": 0.9884375 - } - }, - "auroc": 0.9874050699869792 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3163, - "fn": 37, - "accuracy": 0.9884375 - } - }, - "auroc": 0.9874050699869792 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1541, - "fn": 59, - "accuracy": 0.963125 - }, - "0.01": { - "tp": 1487, - "fn": 113, - "accuracy": 0.929375 - } - }, - "auroc": 0.9706578287760417 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1541, - "fn": 59, - "accuracy": 0.963125 - }, - "0.01": { - "tp": 1487, - "fn": 113, - "accuracy": 0.929375 - } - }, - "auroc": 0.9706578287760417 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1489, - "fn": 111, - "accuracy": 0.930625 - }, - "0.01": { - "tp": 1401, - "fn": 199, - "accuracy": 0.875625 - } - }, - "auroc": 0.95229658203125 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 1489, - "fn": 111, - "accuracy": 0.930625 - }, - "0.01": { - "tp": 1401, - "fn": 199, - "accuracy": 0.875625 - } - }, - "auroc": 0.95229658203125 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3030, - "fn": 170, - "accuracy": 0.946875 - }, - "0.01": { - "tp": 2888, - "fn": 312, - "accuracy": 0.9025 - } - }, - "auroc": 0.9614772054036458 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 3030, - "fn": 170, - "accuracy": 0.946875 - }, - "0.01": { - "tp": 2888, - "fn": 312, - "accuracy": 0.9025 - } - }, - "auroc": 0.9614772054036458 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 17222, - "fn": 378, - "accuracy": 0.9785227272727273 - }, - "0.01": { - "tp": 16744, - "fn": 856, - "accuracy": 0.9513636363636364 - } - }, - "auroc": 0.9781990885416667 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 9549, - "fn": 51, - "accuracy": 0.9946875 - }, - "0.01": { - "tp": 9477, - "fn": 123, - "accuracy": 0.9871875 - } - }, - "auroc": 0.9909814724392362 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 26771, - "fn": 429, - "accuracy": 0.9842279411764706 - }, - "0.01": { - "tp": 26221, - "fn": 979, - "accuracy": 0.9640073529411765 - } - }, - "auroc": 0.9827105181525735 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 16964, - "fn": 636, - "accuracy": 0.9638636363636364 - }, - "0.01": { - "tp": 16227, - "fn": 1373, - "accuracy": 0.9219886363636364 - } - }, - "auroc": 0.9659882501775569 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 9390, - "fn": 210, - "accuracy": 0.978125 - }, - "0.01": { - "tp": 9105, - "fn": 495, - "accuracy": 0.9484375 - } - }, - "auroc": 0.9713364691840277 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 26354, - "fn": 846, - "accuracy": 0.9688970588235294 - }, - "0.01": { - "tp": 25332, - "fn": 1868, - "accuracy": 0.9313235294117647 - } - }, - "auroc": 0.967875856885723 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 34186, - "fn": 1014, - "accuracy": 0.9711931818181818 - }, - "0.01": { - "tp": 32971, - "fn": 2229, - "accuracy": 0.9366761363636363 - } - }, - "auroc": 0.9720936693596118 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 18939, - "fn": 261, - "accuracy": 0.98640625 - }, - "0.01": { - "tp": 18582, - "fn": 618, - "accuracy": 0.9678125 - } - }, - "auroc": 0.981158970811632 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "perplexity_misspelling", - "accuracy": { - "0.05": { - "tp": 53125, - "fn": 1275, - "accuracy": 0.9765625 - }, - "0.01": { - "tp": 51553, - "fn": 2847, - "accuracy": 0.9476654411764706 - } - }, - "auroc": 0.9752931875191484 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - } - }, - "auroc": 0.995813427734375 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1582, - "fn": 18, - "accuracy": 0.98875 - } - }, - "auroc": 0.9913526367187501 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3193, - "fn": 7, - "accuracy": 0.9978125 - }, - "0.01": { - "tp": 3179, - "fn": 21, - "accuracy": 0.9934375 - } - }, - "auroc": 0.9935830322265625 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1593, - "fn": 7, - "accuracy": 0.995625 - } - }, - "auroc": 0.995637353515625 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1587, - "fn": 13, - "accuracy": 0.991875 - }, - "0.01": { - "tp": 1535, - "fn": 65, - "accuracy": 0.959375 - } - }, - "auroc": 0.9820313639322916 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3186, - "fn": 14, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 3128, - "fn": 72, - "accuracy": 0.9775 - } - }, - "auroc": 0.9888343587239583 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3198, - "fn": 2, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - } - }, - "auroc": 0.995725390625 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3181, - "fn": 19, - "accuracy": 0.9940625 - }, - "0.01": { - "tp": 3117, - "fn": 83, - "accuracy": 0.9740625 - } - }, - "auroc": 0.9866920003255208 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 6379, - "fn": 21, - "accuracy": 0.99671875 - }, - "0.01": { - "tp": 6307, - "fn": 93, - "accuracy": 0.98546875 - } - }, - "auroc": 0.9912086954752605 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1541, - "fn": 59, - "accuracy": 0.963125 - }, - "0.01": { - "tp": 1408, - "fn": 192, - "accuracy": 0.88 - } - }, - "auroc": 0.9547259440104168 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1407, - "fn": 193, - "accuracy": 0.879375 - }, - "0.01": { - "tp": 1073, - "fn": 527, - "accuracy": 0.670625 - } - }, - "auroc": 0.8835445475260417 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2948, - "fn": 252, - "accuracy": 0.92125 - }, - "0.01": { - "tp": 2481, - "fn": 719, - "accuracy": 0.7753125 - } - }, - "auroc": 0.9191352457682291 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1540, - "fn": 60, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 1425, - "fn": 175, - "accuracy": 0.890625 - } - }, - "auroc": 0.9518404459635417 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1194, - "fn": 406, - "accuracy": 0.74625 - }, - "0.01": { - "tp": 821, - "fn": 779, - "accuracy": 0.513125 - } - }, - "auroc": 0.8414669270833333 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2734, - "fn": 466, - "accuracy": 0.854375 - }, - "0.01": { - "tp": 2246, - "fn": 954, - "accuracy": 0.701875 - } - }, - "auroc": 0.8966536865234375 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3081, - "fn": 119, - "accuracy": 0.9628125 - }, - "0.01": { - "tp": 2833, - "fn": 367, - "accuracy": 0.8853125 - } - }, - "auroc": 0.9532831949869791 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2601, - "fn": 599, - "accuracy": 0.8128125 - }, - "0.01": { - "tp": 1894, - "fn": 1306, - "accuracy": 0.591875 - } - }, - "auroc": 0.8625057373046875 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 5682, - "fn": 718, - "accuracy": 0.8878125 - }, - "0.01": { - "tp": 4727, - "fn": 1673, - "accuracy": 0.73859375 - } - }, - "auroc": 0.9078944661458334 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1585, - "fn": 15, - "accuracy": 0.990625 - }, - "0.01": { - "tp": 1518, - "fn": 82, - "accuracy": 0.94875 - } - }, - "auroc": 0.9776152506510416 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1539, - "fn": 61, - "accuracy": 0.961875 - }, - "0.01": { - "tp": 1380, - "fn": 220, - "accuracy": 0.8625 - } - }, - "auroc": 0.9431801595052083 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3124, - "fn": 76, - "accuracy": 0.97625 - }, - "0.01": { - "tp": 2898, - "fn": 302, - "accuracy": 0.905625 - } - }, - "auroc": 0.9603977050781249 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1560, - "fn": 40, - "accuracy": 0.975 - }, - "0.01": { - "tp": 1502, - "fn": 98, - "accuracy": 0.93875 - } - }, - "auroc": 0.9739478678385417 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1480, - "fn": 120, - "accuracy": 0.925 - }, - "0.01": { - "tp": 1237, - "fn": 363, - "accuracy": 0.773125 - } - }, - "auroc": 0.9167321451822916 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3040, - "fn": 160, - "accuracy": 0.95 - }, - "0.01": { - "tp": 2739, - "fn": 461, - "accuracy": 0.8559375 - } - }, - "auroc": 0.9453400065104166 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3145, - "fn": 55, - "accuracy": 0.9828125 - }, - "0.01": { - "tp": 3020, - "fn": 180, - "accuracy": 0.94375 - } - }, - "auroc": 0.9757815592447916 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3019, - "fn": 181, - "accuracy": 0.9434375 - }, - "0.01": { - "tp": 2617, - "fn": 583, - "accuracy": 0.8178125 - } - }, - "auroc": 0.92995615234375 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 6164, - "fn": 236, - "accuracy": 0.963125 - }, - "0.01": { - "tp": 5637, - "fn": 763, - "accuracy": 0.88078125 - } - }, - "auroc": 0.9528688557942708 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1477, - "fn": 123, - "accuracy": 0.923125 - }, - "0.01": { - "tp": 1266, - "fn": 334, - "accuracy": 0.79125 - } - }, - "auroc": 0.9221966145833332 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1574, - "fn": 26, - "accuracy": 0.98375 - }, - "0.01": { - "tp": 1519, - "fn": 81, - "accuracy": 0.949375 - } - }, - "auroc": 0.9712079264322917 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3051, - "fn": 149, - "accuracy": 0.9534375 - }, - "0.01": { - "tp": 2785, - "fn": 415, - "accuracy": 0.8703125 - } - }, - "auroc": 0.9467022705078125 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1540, - "fn": 60, - "accuracy": 0.9625 - }, - "0.01": { - "tp": 1404, - "fn": 196, - "accuracy": 0.8775 - } - }, - "auroc": 0.9491346516927084 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1454, - "fn": 146, - "accuracy": 0.90875 - }, - "0.01": { - "tp": 1123, - "fn": 477, - "accuracy": 0.701875 - } - }, - "auroc": 0.8958622233072916 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2994, - "fn": 206, - "accuracy": 0.935625 - }, - "0.01": { - "tp": 2527, - "fn": 673, - "accuracy": 0.7896875 - } - }, - "auroc": 0.9224984375 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3017, - "fn": 183, - "accuracy": 0.9428125 - }, - "0.01": { - "tp": 2670, - "fn": 530, - "accuracy": 0.834375 - } - }, - "auroc": 0.9356656331380208 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3028, - "fn": 172, - "accuracy": 0.94625 - }, - "0.01": { - "tp": 2642, - "fn": 558, - "accuracy": 0.825625 - } - }, - "auroc": 0.9335350748697917 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 6045, - "fn": 355, - "accuracy": 0.94453125 - }, - "0.01": { - "tp": 5312, - "fn": 1088, - "accuracy": 0.83 - } - }, - "auroc": 0.9346003540039063 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1541, - "fn": 59, - "accuracy": 0.963125 - }, - "0.01": { - "tp": 1424, - "fn": 176, - "accuracy": 0.89 - } - }, - "auroc": 0.9564352213541667 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1501, - "fn": 99, - "accuracy": 0.938125 - }, - "0.01": { - "tp": 1322, - "fn": 278, - "accuracy": 0.82625 - } - }, - "auroc": 0.9270678548177083 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3042, - "fn": 158, - "accuracy": 0.950625 - }, - "0.01": { - "tp": 2746, - "fn": 454, - "accuracy": 0.858125 - } - }, - "auroc": 0.9417515380859375 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1485, - "fn": 115, - "accuracy": 0.928125 - }, - "0.01": { - "tp": 1294, - "fn": 306, - "accuracy": 0.80875 - } - }, - "auroc": 0.9266640136718751 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1226, - "fn": 374, - "accuracy": 0.76625 - }, - "0.01": { - "tp": 806, - "fn": 794, - "accuracy": 0.50375 - } - }, - "auroc": 0.845912646484375 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2711, - "fn": 489, - "accuracy": 0.8471875 - }, - "0.01": { - "tp": 2100, - "fn": 1100, - "accuracy": 0.65625 - } - }, - "auroc": 0.886288330078125 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3026, - "fn": 174, - "accuracy": 0.945625 - }, - "0.01": { - "tp": 2718, - "fn": 482, - "accuracy": 0.849375 - } - }, - "auroc": 0.9415496175130209 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2727, - "fn": 473, - "accuracy": 0.8521875 - }, - "0.01": { - "tp": 2128, - "fn": 1072, - "accuracy": 0.665 - } - }, - "auroc": 0.8864902506510416 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 5753, - "fn": 647, - "accuracy": 0.89890625 - }, - "0.01": { - "tp": 4846, - "fn": 1554, - "accuracy": 0.7571875 - } - }, - "auroc": 0.9140199340820312 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - }, - "0.01": { - "tp": 1580, - "fn": 20, - "accuracy": 0.9875 - } - }, - "auroc": 0.9906379557291667 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1593, - "fn": 7, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 1559, - "fn": 41, - "accuracy": 0.974375 - } - }, - "auroc": 0.98451494140625 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3185, - "fn": 15, - "accuracy": 0.9953125 - }, - "0.01": { - "tp": 3139, - "fn": 61, - "accuracy": 0.9809375 - } - }, - "auroc": 0.9875764485677083 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1575, - "fn": 25, - "accuracy": 0.984375 - }, - "0.01": { - "tp": 1549, - "fn": 51, - "accuracy": 0.968125 - } - }, - "auroc": 0.9836405273437501 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1549, - "fn": 51, - "accuracy": 0.968125 - }, - "0.01": { - "tp": 1447, - "fn": 153, - "accuracy": 0.904375 - } - }, - "auroc": 0.959482373046875 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3124, - "fn": 76, - "accuracy": 0.97625 - }, - "0.01": { - "tp": 2996, - "fn": 204, - "accuracy": 0.93625 - } - }, - "auroc": 0.9715614501953127 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3167, - "fn": 33, - "accuracy": 0.9896875 - }, - "0.01": { - "tp": 3129, - "fn": 71, - "accuracy": 0.9778125 - } - }, - "auroc": 0.9871392415364584 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3142, - "fn": 58, - "accuracy": 0.981875 - }, - "0.01": { - "tp": 3006, - "fn": 194, - "accuracy": 0.939375 - } - }, - "auroc": 0.9719986572265626 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 6309, - "fn": 91, - "accuracy": 0.98578125 - }, - "0.01": { - "tp": 6135, - "fn": 265, - "accuracy": 0.95859375 - } - }, - "auroc": 0.9795689493815104 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1493, - "fn": 107, - "accuracy": 0.933125 - }, - "0.01": { - "tp": 1343, - "fn": 257, - "accuracy": 0.839375 - } - }, - "auroc": 0.9369581868489583 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1493, - "fn": 107, - "accuracy": 0.933125 - }, - "0.01": { - "tp": 1343, - "fn": 257, - "accuracy": 0.839375 - } - }, - "auroc": 0.9369581868489583 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1466, - "fn": 134, - "accuracy": 0.91625 - }, - "0.01": { - "tp": 1310, - "fn": 290, - "accuracy": 0.81875 - } - }, - "auroc": 0.9303712890625 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1466, - "fn": 134, - "accuracy": 0.91625 - }, - "0.01": { - "tp": 1310, - "fn": 290, - "accuracy": 0.81875 - } - }, - "auroc": 0.9303712890625 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2959, - "fn": 241, - "accuracy": 0.9246875 - }, - "0.01": { - "tp": 2653, - "fn": 547, - "accuracy": 0.8290625 - } - }, - "auroc": 0.933664737955729 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2959, - "fn": 241, - "accuracy": 0.9246875 - }, - "0.01": { - "tp": 2653, - "fn": 547, - "accuracy": 0.8290625 - } - }, - "auroc": 0.933664737955729 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1503, - "fn": 97, - "accuracy": 0.939375 - }, - "0.01": { - "tp": 1355, - "fn": 245, - "accuracy": 0.846875 - } - }, - "auroc": 0.9371241699218749 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1503, - "fn": 97, - "accuracy": 0.939375 - }, - "0.01": { - "tp": 1355, - "fn": 245, - "accuracy": 0.846875 - } - }, - "auroc": 0.9371241699218749 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1450, - "fn": 150, - "accuracy": 0.90625 - }, - "0.01": { - "tp": 1279, - "fn": 321, - "accuracy": 0.799375 - } - }, - "auroc": 0.9236302571614584 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1450, - "fn": 150, - "accuracy": 0.90625 - }, - "0.01": { - "tp": 1279, - "fn": 321, - "accuracy": 0.799375 - } - }, - "auroc": 0.9236302571614584 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2953, - "fn": 247, - "accuracy": 0.9228125 - }, - "0.01": { - "tp": 2634, - "fn": 566, - "accuracy": 0.823125 - } - }, - "auroc": 0.9303772135416666 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 2953, - "fn": 247, - "accuracy": 0.9228125 - }, - "0.01": { - "tp": 2634, - "fn": 566, - "accuracy": 0.823125 - } - }, - "auroc": 0.9303772135416666 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - } - }, - "auroc": 0.992444580078125 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - } - }, - "auroc": 0.992444580078125 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1575, - "fn": 25, - "accuracy": 0.984375 - } - }, - "auroc": 0.990785107421875 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1575, - "fn": 25, - "accuracy": 0.984375 - } - }, - "auroc": 0.990785107421875 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3188, - "fn": 12, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 3159, - "fn": 41, - "accuracy": 0.9871875 - } - }, - "auroc": 0.99161484375 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3188, - "fn": 12, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 3159, - "fn": 41, - "accuracy": 0.9871875 - } - }, - "auroc": 0.99161484375 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - } - }, - "auroc": 0.99266337890625 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - } - }, - "auroc": 0.99266337890625 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1587, - "fn": 13, - "accuracy": 0.991875 - }, - "0.01": { - "tp": 1548, - "fn": 52, - "accuracy": 0.9675 - } - }, - "auroc": 0.9791282226562499 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1587, - "fn": 13, - "accuracy": 0.991875 - }, - "0.01": { - "tp": 1548, - "fn": 52, - "accuracy": 0.9675 - } - }, - "auroc": 0.9791282226562499 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3184, - "fn": 16, - "accuracy": 0.995 - }, - "0.01": { - "tp": 3137, - "fn": 63, - "accuracy": 0.9803125 - } - }, - "auroc": 0.9858958007812499 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3184, - "fn": 16, - "accuracy": 0.995 - }, - "0.01": { - "tp": 3137, - "fn": 63, - "accuracy": 0.9803125 - } - }, - "auroc": 0.9858958007812499 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1520, - "fn": 80, - "accuracy": 0.95 - }, - "0.01": { - "tp": 1409, - "fn": 191, - "accuracy": 0.880625 - } - }, - "auroc": 0.9571074707031251 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1520, - "fn": 80, - "accuracy": 0.95 - }, - "0.01": { - "tp": 1409, - "fn": 191, - "accuracy": 0.880625 - } - }, - "auroc": 0.9571074707031251 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1503, - "fn": 97, - "accuracy": 0.939375 - }, - "0.01": { - "tp": 1361, - "fn": 239, - "accuracy": 0.850625 - } - }, - "auroc": 0.9443228190104167 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 1503, - "fn": 97, - "accuracy": 0.939375 - }, - "0.01": { - "tp": 1361, - "fn": 239, - "accuracy": 0.850625 - } - }, - "auroc": 0.9443228190104167 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3023, - "fn": 177, - "accuracy": 0.9446875 - }, - "0.01": { - "tp": 2770, - "fn": 430, - "accuracy": 0.865625 - } - }, - "auroc": 0.9507151448567709 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 3023, - "fn": 177, - "accuracy": 0.9446875 - }, - "0.01": { - "tp": 2770, - "fn": 430, - "accuracy": 0.865625 - } - }, - "auroc": 0.9507151448567709 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 17042, - "fn": 558, - "accuracy": 0.9682954545454545 - }, - "0.01": { - "tp": 16073, - "fn": 1527, - "accuracy": 0.9132386363636363 - } - }, - "auroc": 0.9648838364109849 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 9208, - "fn": 392, - "accuracy": 0.9591666666666666 - }, - "0.01": { - "tp": 8435, - "fn": 1165, - "accuracy": 0.8786458333333333 - } - }, - "auroc": 0.950144677734375 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 26250, - "fn": 950, - "accuracy": 0.9650735294117647 - }, - "0.01": { - "tp": 24508, - "fn": 2692, - "accuracy": 0.9010294117647059 - } - }, - "auroc": 0.9596817804074755 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 16899, - "fn": 701, - "accuracy": 0.9601704545454546 - }, - "0.01": { - "tp": 15840, - "fn": 1760, - "accuracy": 0.9 - } - }, - "auroc": 0.9590093232125947 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 8490, - "fn": 1110, - "accuracy": 0.884375 - }, - "0.01": { - "tp": 6969, - "fn": 2631, - "accuracy": 0.7259375 - } - }, - "auroc": 0.9069146131727431 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 25389, - "fn": 1811, - "accuracy": 0.9334191176470589 - }, - "0.01": { - "tp": 22809, - "fn": 4391, - "accuracy": 0.8385661764705883 - } - }, - "auroc": 0.9406229549632353 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 33941, - "fn": 1259, - "accuracy": 0.9642329545454545 - }, - "0.01": { - "tp": 31913, - "fn": 3287, - "accuracy": 0.9066193181818182 - } - }, - "auroc": 0.9619465798117898 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 17698, - "fn": 1502, - "accuracy": 0.9217708333333333 - }, - "0.01": { - "tp": 15404, - "fn": 3796, - "accuracy": 0.8022916666666666 - } - }, - "auroc": 0.928529645453559 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "paraphrase", - "accuracy": { - "0.05": { - "tp": 51639, - "fn": 2761, - "accuracy": 0.9492463235294117 - }, - "0.01": { - "tp": 47317, - "fn": 7083, - "accuracy": 0.869797794117647 - } - }, - "auroc": 0.9501523676853555 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - } - }, - "auroc": 0.9967869303385416 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - } - }, - "auroc": 0.9951549967447917 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3198, - "fn": 2, - "accuracy": 0.999375 - } - }, - "auroc": 0.9959709635416667 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - } - }, - "auroc": 0.9959591959635417 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - } - }, - "auroc": 0.9906822916666667 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3199, - "fn": 1, - "accuracy": 0.9996875 - }, - "0.01": { - "tp": 3181, - "fn": 19, - "accuracy": 0.9940625 - } - }, - "auroc": 0.9933207438151042 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3196, - "fn": 4, - "accuracy": 0.99875 - } - }, - "auroc": 0.9963730631510417 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3199, - "fn": 1, - "accuracy": 0.9996875 - }, - "0.01": { - "tp": 3183, - "fn": 17, - "accuracy": 0.9946875 - } - }, - "auroc": 0.9929186442057292 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 6399, - "fn": 1, - "accuracy": 0.99984375 - }, - "0.01": { - "tp": 6379, - "fn": 21, - "accuracy": 0.99671875 - } - }, - "auroc": 0.9946458536783854 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1565, - "fn": 35, - "accuracy": 0.978125 - }, - "0.01": { - "tp": 1492, - "fn": 108, - "accuracy": 0.9325 - } - }, - "auroc": 0.9731423502604167 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - }, - "0.01": { - "tp": 1554, - "fn": 46, - "accuracy": 0.97125 - } - }, - "auroc": 0.9872086588541666 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3141, - "fn": 59, - "accuracy": 0.9815625 - }, - "0.01": { - "tp": 3046, - "fn": 154, - "accuracy": 0.951875 - } - }, - "auroc": 0.9801755045572917 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1500, - "fn": 100, - "accuracy": 0.9375 - }, - "0.01": { - "tp": 1367, - "fn": 233, - "accuracy": 0.854375 - } - }, - "auroc": 0.9421653645833333 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1535, - "fn": 65, - "accuracy": 0.959375 - }, - "0.01": { - "tp": 1481, - "fn": 119, - "accuracy": 0.925625 - } - }, - "auroc": 0.9661674967447916 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3035, - "fn": 165, - "accuracy": 0.9484375 - }, - "0.01": { - "tp": 2848, - "fn": 352, - "accuracy": 0.89 - } - }, - "auroc": 0.9541664306640625 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3065, - "fn": 135, - "accuracy": 0.9578125 - }, - "0.01": { - "tp": 2859, - "fn": 341, - "accuracy": 0.8934375 - } - }, - "auroc": 0.9576538574218751 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3111, - "fn": 89, - "accuracy": 0.9721875 - }, - "0.01": { - "tp": 3035, - "fn": 165, - "accuracy": 0.9484375 - } - }, - "auroc": 0.9766880777994792 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 6176, - "fn": 224, - "accuracy": 0.965 - }, - "0.01": { - "tp": 5894, - "fn": 506, - "accuracy": 0.9209375 - } - }, - "auroc": 0.9671709676106771 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 1585, - "fn": 15, - "accuracy": 0.990625 - } - }, - "auroc": 0.9930502604166667 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 1579, - "fn": 21, - "accuracy": 0.986875 - } - }, - "auroc": 0.986801708984375 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3181, - "fn": 19, - "accuracy": 0.9940625 - }, - "0.01": { - "tp": 3164, - "fn": 36, - "accuracy": 0.98875 - } - }, - "auroc": 0.9899259847005208 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 1579, - "fn": 21, - "accuracy": 0.986875 - } - }, - "auroc": 0.9910825032552083 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - }, - "0.01": { - "tp": 1525, - "fn": 75, - "accuracy": 0.953125 - } - }, - "auroc": 0.9704899088541667 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3175, - "fn": 25, - "accuracy": 0.9921875 - }, - "0.01": { - "tp": 3104, - "fn": 96, - "accuracy": 0.97 - } - }, - "auroc": 0.9807862060546875 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3182, - "fn": 18, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 3164, - "fn": 36, - "accuracy": 0.98875 - } - }, - "auroc": 0.9920663818359375 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3174, - "fn": 26, - "accuracy": 0.991875 - }, - "0.01": { - "tp": 3104, - "fn": 96, - "accuracy": 0.97 - } - }, - "auroc": 0.9786458089192709 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 6356, - "fn": 44, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 6268, - "fn": 132, - "accuracy": 0.979375 - } - }, - "auroc": 0.9853560953776042 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1517, - "fn": 83, - "accuracy": 0.948125 - }, - "0.01": { - "tp": 1399, - "fn": 201, - "accuracy": 0.874375 - } - }, - "auroc": 0.9521425944010418 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1574, - "fn": 26, - "accuracy": 0.98375 - } - }, - "auroc": 0.9917976888020833 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3111, - "fn": 89, - "accuracy": 0.9721875 - }, - "0.01": { - "tp": 2973, - "fn": 227, - "accuracy": 0.9290625 - } - }, - "auroc": 0.9719701416015625 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1533, - "fn": 67, - "accuracy": 0.958125 - }, - "0.01": { - "tp": 1343, - "fn": 257, - "accuracy": 0.839375 - } - }, - "auroc": 0.942598583984375 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1556, - "fn": 44, - "accuracy": 0.9725 - }, - "0.01": { - "tp": 1498, - "fn": 102, - "accuracy": 0.93625 - } - }, - "auroc": 0.9576928548177083 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3089, - "fn": 111, - "accuracy": 0.9653125 - }, - "0.01": { - "tp": 2841, - "fn": 359, - "accuracy": 0.8878125 - } - }, - "auroc": 0.9501457194010416 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3050, - "fn": 150, - "accuracy": 0.953125 - }, - "0.01": { - "tp": 2742, - "fn": 458, - "accuracy": 0.856875 - } - }, - "auroc": 0.9473705891927084 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3150, - "fn": 50, - "accuracy": 0.984375 - }, - "0.01": { - "tp": 3072, - "fn": 128, - "accuracy": 0.96 - } - }, - "auroc": 0.9747452718098957 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 6200, - "fn": 200, - "accuracy": 0.96875 - }, - "0.01": { - "tp": 5814, - "fn": 586, - "accuracy": 0.9084375 - } - }, - "auroc": 0.9610579305013021 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1565, - "fn": 35, - "accuracy": 0.978125 - }, - "0.01": { - "tp": 1504, - "fn": 96, - "accuracy": 0.94 - } - }, - "auroc": 0.9793320800781251 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - }, - "0.01": { - "tp": 1578, - "fn": 22, - "accuracy": 0.98625 - } - }, - "auroc": 0.9869983072916666 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3157, - "fn": 43, - "accuracy": 0.9865625 - }, - "0.01": { - "tp": 3082, - "fn": 118, - "accuracy": 0.963125 - } - }, - "auroc": 0.983165193684896 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1424, - "fn": 176, - "accuracy": 0.89 - }, - "0.01": { - "tp": 1217, - "fn": 383, - "accuracy": 0.760625 - } - }, - "auroc": 0.9152535156249999 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1545, - "fn": 55, - "accuracy": 0.965625 - }, - "0.01": { - "tp": 1453, - "fn": 147, - "accuracy": 0.908125 - } - }, - "auroc": 0.953453173828125 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2969, - "fn": 231, - "accuracy": 0.9278125 - }, - "0.01": { - "tp": 2670, - "fn": 530, - "accuracy": 0.834375 - } - }, - "auroc": 0.9343533447265624 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2989, - "fn": 211, - "accuracy": 0.9340625 - }, - "0.01": { - "tp": 2721, - "fn": 479, - "accuracy": 0.8503125 - } - }, - "auroc": 0.9472927978515625 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3137, - "fn": 63, - "accuracy": 0.9803125 - }, - "0.01": { - "tp": 3031, - "fn": 169, - "accuracy": 0.9471875 - } - }, - "auroc": 0.970225740559896 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 6126, - "fn": 274, - "accuracy": 0.9571875 - }, - "0.01": { - "tp": 5752, - "fn": 648, - "accuracy": 0.89875 - } - }, - "auroc": 0.9587592692057292 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9955173014322916 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1593, - "fn": 7, - "accuracy": 0.995625 - } - }, - "auroc": 0.9943202311197917 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3193, - "fn": 7, - "accuracy": 0.9978125 - }, - "0.01": { - "tp": 3189, - "fn": 11, - "accuracy": 0.9965625 - } - }, - "auroc": 0.9949187662760417 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 1586, - "fn": 14, - "accuracy": 0.99125 - } - }, - "auroc": 0.99273515625 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1581, - "fn": 19, - "accuracy": 0.988125 - }, - "0.01": { - "tp": 1546, - "fn": 54, - "accuracy": 0.96625 - } - }, - "auroc": 0.979903369140625 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3172, - "fn": 28, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 3132, - "fn": 68, - "accuracy": 0.97875 - } - }, - "auroc": 0.9863192626953126 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3187, - "fn": 13, - "accuracy": 0.9959375 - }, - "0.01": { - "tp": 3182, - "fn": 18, - "accuracy": 0.994375 - } - }, - "auroc": 0.9941262288411459 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3178, - "fn": 22, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 3139, - "fn": 61, - "accuracy": 0.9809375 - } - }, - "auroc": 0.9871118001302084 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 6365, - "fn": 35, - "accuracy": 0.99453125 - }, - "0.01": { - "tp": 6321, - "fn": 79, - "accuracy": 0.98765625 - } - }, - "auroc": 0.990619014485677 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1548, - "fn": 52, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 1503, - "fn": 97, - "accuracy": 0.939375 - } - }, - "auroc": 0.9709605143229167 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1548, - "fn": 52, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 1503, - "fn": 97, - "accuracy": 0.939375 - } - }, - "auroc": 0.9709605143229167 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1535, - "fn": 65, - "accuracy": 0.959375 - }, - "0.01": { - "tp": 1465, - "fn": 135, - "accuracy": 0.915625 - } - }, - "auroc": 0.9619362141927084 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1535, - "fn": 65, - "accuracy": 0.959375 - }, - "0.01": { - "tp": 1465, - "fn": 135, - "accuracy": 0.915625 - } - }, - "auroc": 0.9619362141927084 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3083, - "fn": 117, - "accuracy": 0.9634375 - }, - "0.01": { - "tp": 2968, - "fn": 232, - "accuracy": 0.9275 - } - }, - "auroc": 0.9664483642578126 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3083, - "fn": 117, - "accuracy": 0.9634375 - }, - "0.01": { - "tp": 2968, - "fn": 232, - "accuracy": 0.9275 - } - }, - "auroc": 0.9664483642578126 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1489, - "fn": 111, - "accuracy": 0.930625 - }, - "0.01": { - "tp": 1367, - "fn": 233, - "accuracy": 0.854375 - } - }, - "auroc": 0.9397853190104166 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1489, - "fn": 111, - "accuracy": 0.930625 - }, - "0.01": { - "tp": 1367, - "fn": 233, - "accuracy": 0.854375 - } - }, - "auroc": 0.9397853190104166 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1417, - "fn": 183, - "accuracy": 0.885625 - }, - "0.01": { - "tp": 1261, - "fn": 339, - "accuracy": 0.788125 - } - }, - "auroc": 0.9173675944010417 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1417, - "fn": 183, - "accuracy": 0.885625 - }, - "0.01": { - "tp": 1261, - "fn": 339, - "accuracy": 0.788125 - } - }, - "auroc": 0.9173675944010417 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2906, - "fn": 294, - "accuracy": 0.908125 - }, - "0.01": { - "tp": 2628, - "fn": 572, - "accuracy": 0.82125 - } - }, - "auroc": 0.9285764567057292 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 2906, - "fn": 294, - "accuracy": 0.908125 - }, - "0.01": { - "tp": 2628, - "fn": 572, - "accuracy": 0.82125 - } - }, - "auroc": 0.9285764567057292 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9925000325520834 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9925000325520834 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - } - }, - "auroc": 0.9902887369791666 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - } - }, - "auroc": 0.9902887369791666 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3187, - "fn": 13, - "accuracy": 0.9959375 - } - }, - "auroc": 0.991394384765625 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3187, - "fn": 13, - "accuracy": 0.9959375 - } - }, - "auroc": 0.991394384765625 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9936864583333334 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9936864583333334 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 1553, - "fn": 47, - "accuracy": 0.970625 - } - }, - "auroc": 0.9777284505208333 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 1553, - "fn": 47, - "accuracy": 0.970625 - } - }, - "auroc": 0.9777284505208333 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3187, - "fn": 13, - "accuracy": 0.9959375 - }, - "0.01": { - "tp": 3148, - "fn": 52, - "accuracy": 0.98375 - } - }, - "auroc": 0.9857074544270833 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3187, - "fn": 13, - "accuracy": 0.9959375 - }, - "0.01": { - "tp": 3148, - "fn": 52, - "accuracy": 0.98375 - } - }, - "auroc": 0.9857074544270833 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1530, - "fn": 70, - "accuracy": 0.95625 - }, - "0.01": { - "tp": 1469, - "fn": 131, - "accuracy": 0.918125 - } - }, - "auroc": 0.9673820475260416 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1530, - "fn": 70, - "accuracy": 0.95625 - }, - "0.01": { - "tp": 1469, - "fn": 131, - "accuracy": 0.918125 - } - }, - "auroc": 0.9673820475260416 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1479, - "fn": 121, - "accuracy": 0.924375 - }, - "0.01": { - "tp": 1381, - "fn": 219, - "accuracy": 0.863125 - } - }, - "auroc": 0.9488975748697916 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 1479, - "fn": 121, - "accuracy": 0.924375 - }, - "0.01": { - "tp": 1381, - "fn": 219, - "accuracy": 0.863125 - } - }, - "auroc": 0.9488975748697916 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3009, - "fn": 191, - "accuracy": 0.9403125 - }, - "0.01": { - "tp": 2850, - "fn": 350, - "accuracy": 0.890625 - } - }, - "auroc": 0.9581398111979167 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 3009, - "fn": 191, - "accuracy": 0.9403125 - }, - "0.01": { - "tp": 2850, - "fn": 350, - "accuracy": 0.890625 - } - }, - "auroc": 0.9581398111979167 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 17194, - "fn": 406, - "accuracy": 0.9769318181818182 - }, - "0.01": { - "tp": 16705, - "fn": 895, - "accuracy": 0.9491477272727272 - } - }, - "auroc": 0.977662353515625 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 9549, - "fn": 51, - "accuracy": 0.9946875 - }, - "0.01": { - "tp": 9477, - "fn": 123, - "accuracy": 0.9871875 - } - }, - "auroc": 0.9903802652994791 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 26743, - "fn": 457, - "accuracy": 0.9831985294117647 - }, - "0.01": { - "tp": 26182, - "fn": 1018, - "accuracy": 0.9625735294117647 - } - }, - "auroc": 0.9821510282628677 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 16854, - "fn": 746, - "accuracy": 0.9576136363636364 - }, - "0.01": { - "tp": 15940, - "fn": 1660, - "accuracy": 0.9056818181818181 - } - }, - "auroc": 0.9614557173295455 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 9400, - "fn": 200, - "accuracy": 0.9791666666666666 - }, - "0.01": { - "tp": 9087, - "fn": 513, - "accuracy": 0.9465625 - } - }, - "auroc": 0.9697315158420139 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 26254, - "fn": 946, - "accuracy": 0.9652205882352941 - }, - "0.01": { - "tp": 25027, - "fn": 2173, - "accuracy": 0.9201102941176471 - } - }, - "auroc": 0.9643765873927697 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 34048, - "fn": 1152, - "accuracy": 0.9672727272727273 - }, - "0.01": { - "tp": 32645, - "fn": 2555, - "accuracy": 0.9274147727272727 - } - }, - "auroc": 0.9695590354225854 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 18949, - "fn": 251, - "accuracy": 0.9869270833333333 - }, - "0.01": { - "tp": 18564, - "fn": 636, - "accuracy": 0.966875 - } - }, - "auroc": 0.9800558905707465 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "number", - "accuracy": { - "0.05": { - "tp": 52997, - "fn": 1403, - "accuracy": 0.9742095588235294 - }, - "0.01": { - "tp": 51209, - "fn": 3191, - "accuracy": 0.9413419117647058 - } - }, - "auroc": 0.9732638078278186 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - } - }, - "auroc": 0.998609130859375 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - } - }, - "auroc": 0.997159326171875 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3198, - "fn": 2, - "accuracy": 0.999375 - } - }, - "auroc": 0.9978842285156251 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - } - }, - "auroc": 0.9982106608072917 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1580, - "fn": 20, - "accuracy": 0.9875 - } - }, - "auroc": 0.9920614583333334 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3197, - "fn": 3, - "accuracy": 0.9990625 - }, - "0.01": { - "tp": 3177, - "fn": 23, - "accuracy": 0.9928125 - } - }, - "auroc": 0.9951360595703125 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3196, - "fn": 4, - "accuracy": 0.99875 - } - }, - "auroc": 0.9984098958333333 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3197, - "fn": 3, - "accuracy": 0.9990625 - }, - "0.01": { - "tp": 3179, - "fn": 21, - "accuracy": 0.9934375 - } - }, - "auroc": 0.9946103922526042 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 6397, - "fn": 3, - "accuracy": 0.99953125 - }, - "0.01": { - "tp": 6375, - "fn": 25, - "accuracy": 0.99609375 - } - }, - "auroc": 0.9965101440429688 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1564, - "fn": 36, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 1491, - "fn": 109, - "accuracy": 0.931875 - } - }, - "auroc": 0.9718000162760417 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - }, - "0.01": { - "tp": 1553, - "fn": 47, - "accuracy": 0.970625 - } - }, - "auroc": 0.9869871256510416 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3140, - "fn": 60, - "accuracy": 0.98125 - }, - "0.01": { - "tp": 3044, - "fn": 156, - "accuracy": 0.95125 - } - }, - "auroc": 0.9793935709635417 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1452, - "fn": 148, - "accuracy": 0.9075 - }, - "0.01": { - "tp": 1294, - "fn": 306, - "accuracy": 0.80875 - } - }, - "auroc": 0.925184521484375 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1534, - "fn": 66, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 1481, - "fn": 119, - "accuracy": 0.925625 - } - }, - "auroc": 0.9656543945312499 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2986, - "fn": 214, - "accuracy": 0.933125 - }, - "0.01": { - "tp": 2775, - "fn": 425, - "accuracy": 0.8671875 - } - }, - "auroc": 0.9454194580078125 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3016, - "fn": 184, - "accuracy": 0.9425 - }, - "0.01": { - "tp": 2785, - "fn": 415, - "accuracy": 0.8703125 - } - }, - "auroc": 0.9484922688802082 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3110, - "fn": 90, - "accuracy": 0.971875 - }, - "0.01": { - "tp": 3034, - "fn": 166, - "accuracy": 0.948125 - } - }, - "auroc": 0.9763207600911457 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 6126, - "fn": 274, - "accuracy": 0.9571875 - }, - "0.01": { - "tp": 5819, - "fn": 581, - "accuracy": 0.90921875 - } - }, - "auroc": 0.962406514485677 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - } - }, - "auroc": 0.99545966796875 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 1578, - "fn": 22, - "accuracy": 0.98625 - } - }, - "auroc": 0.9883959798177083 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3182, - "fn": 18, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 3162, - "fn": 38, - "accuracy": 0.988125 - } - }, - "auroc": 0.9919278238932292 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1588, - "fn": 12, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 1575, - "fn": 25, - "accuracy": 0.984375 - } - }, - "auroc": 0.9929184895833333 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - }, - "0.01": { - "tp": 1527, - "fn": 73, - "accuracy": 0.954375 - } - }, - "auroc": 0.9708409667968749 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3172, - "fn": 28, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 3102, - "fn": 98, - "accuracy": 0.969375 - } - }, - "auroc": 0.9818797281901042 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3179, - "fn": 21, - "accuracy": 0.9934375 - }, - "0.01": { - "tp": 3159, - "fn": 41, - "accuracy": 0.9871875 - } - }, - "auroc": 0.9941890787760417 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3175, - "fn": 25, - "accuracy": 0.9921875 - }, - "0.01": { - "tp": 3105, - "fn": 95, - "accuracy": 0.9703125 - } - }, - "auroc": 0.9796184733072917 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 6354, - "fn": 46, - "accuracy": 0.9928125 - }, - "0.01": { - "tp": 6264, - "fn": 136, - "accuracy": 0.97875 - } - }, - "auroc": 0.9869037760416667 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1518, - "fn": 82, - "accuracy": 0.94875 - }, - "0.01": { - "tp": 1404, - "fn": 196, - "accuracy": 0.8775 - } - }, - "auroc": 0.95214189453125 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 1571, - "fn": 29, - "accuracy": 0.981875 - } - }, - "auroc": 0.991532568359375 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3113, - "fn": 87, - "accuracy": 0.9728125 - }, - "0.01": { - "tp": 2975, - "fn": 225, - "accuracy": 0.9296875 - } - }, - "auroc": 0.9718372314453125 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1510, - "fn": 90, - "accuracy": 0.94375 - }, - "0.01": { - "tp": 1282, - "fn": 318, - "accuracy": 0.80125 - } - }, - "auroc": 0.9324824544270833 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1550, - "fn": 50, - "accuracy": 0.96875 - }, - "0.01": { - "tp": 1484, - "fn": 116, - "accuracy": 0.9275 - } - }, - "auroc": 0.9522420084635417 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3060, - "fn": 140, - "accuracy": 0.95625 - }, - "0.01": { - "tp": 2766, - "fn": 434, - "accuracy": 0.864375 - } - }, - "auroc": 0.9423622314453124 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3028, - "fn": 172, - "accuracy": 0.94625 - }, - "0.01": { - "tp": 2686, - "fn": 514, - "accuracy": 0.839375 - } - }, - "auroc": 0.9423121744791667 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3145, - "fn": 55, - "accuracy": 0.9828125 - }, - "0.01": { - "tp": 3055, - "fn": 145, - "accuracy": 0.9546875 - } - }, - "auroc": 0.9718872884114583 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 6173, - "fn": 227, - "accuracy": 0.96453125 - }, - "0.01": { - "tp": 5741, - "fn": 659, - "accuracy": 0.89703125 - } - }, - "auroc": 0.9570997314453125 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1565, - "fn": 35, - "accuracy": 0.978125 - }, - "0.01": { - "tp": 1502, - "fn": 98, - "accuracy": 0.93875 - } - }, - "auroc": 0.9784921712239583 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1569, - "fn": 31, - "accuracy": 0.980625 - } - }, - "auroc": 0.9847839192708333 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3154, - "fn": 46, - "accuracy": 0.985625 - }, - "0.01": { - "tp": 3071, - "fn": 129, - "accuracy": 0.9596875 - } - }, - "auroc": 0.9816380452473958 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1364, - "fn": 236, - "accuracy": 0.8525 - }, - "0.01": { - "tp": 1126, - "fn": 474, - "accuracy": 0.70375 - } - }, - "auroc": 0.8968723307291666 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1533, - "fn": 67, - "accuracy": 0.958125 - }, - "0.01": { - "tp": 1432, - "fn": 168, - "accuracy": 0.895 - } - }, - "auroc": 0.9479154134114585 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2897, - "fn": 303, - "accuracy": 0.9053125 - }, - "0.01": { - "tp": 2558, - "fn": 642, - "accuracy": 0.799375 - } - }, - "auroc": 0.9223938720703125 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2929, - "fn": 271, - "accuracy": 0.9153125 - }, - "0.01": { - "tp": 2628, - "fn": 572, - "accuracy": 0.82125 - } - }, - "auroc": 0.9376822509765625 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3122, - "fn": 78, - "accuracy": 0.975625 - }, - "0.01": { - "tp": 3001, - "fn": 199, - "accuracy": 0.9378125 - } - }, - "auroc": 0.9663496663411457 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 6051, - "fn": 349, - "accuracy": 0.94546875 - }, - "0.01": { - "tp": 5629, - "fn": 771, - "accuracy": 0.87953125 - } - }, - "auroc": 0.952015958658854 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.997239501953125 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - } - }, - "auroc": 0.9971789225260418 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3193, - "fn": 7, - "accuracy": 0.9978125 - }, - "0.01": { - "tp": 3189, - "fn": 11, - "accuracy": 0.9965625 - } - }, - "auroc": 0.9972092122395833 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1588, - "fn": 12, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 1582, - "fn": 18, - "accuracy": 0.98875 - } - }, - "auroc": 0.9934367350260418 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1563, - "fn": 37, - "accuracy": 0.976875 - }, - "0.01": { - "tp": 1526, - "fn": 74, - "accuracy": 0.95375 - } - }, - "auroc": 0.9781484863281249 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3151, - "fn": 49, - "accuracy": 0.9846875 - }, - "0.01": { - "tp": 3108, - "fn": 92, - "accuracy": 0.97125 - } - }, - "auroc": 0.9857926106770832 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3184, - "fn": 16, - "accuracy": 0.995 - }, - "0.01": { - "tp": 3177, - "fn": 23, - "accuracy": 0.9928125 - } - }, - "auroc": 0.9953381184895833 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3160, - "fn": 40, - "accuracy": 0.9875 - }, - "0.01": { - "tp": 3120, - "fn": 80, - "accuracy": 0.975 - } - }, - "auroc": 0.9876637044270834 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 6344, - "fn": 56, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 6297, - "fn": 103, - "accuracy": 0.98390625 - } - }, - "auroc": 0.9915009114583334 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1547, - "fn": 53, - "accuracy": 0.966875 - }, - "0.01": { - "tp": 1490, - "fn": 110, - "accuracy": 0.93125 - } - }, - "auroc": 0.9676311197916666 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1547, - "fn": 53, - "accuracy": 0.966875 - }, - "0.01": { - "tp": 1490, - "fn": 110, - "accuracy": 0.93125 - } - }, - "auroc": 0.9676311197916666 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1526, - "fn": 74, - "accuracy": 0.95375 - }, - "0.01": { - "tp": 1450, - "fn": 150, - "accuracy": 0.90625 - } - }, - "auroc": 0.9579713867187499 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1526, - "fn": 74, - "accuracy": 0.95375 - }, - "0.01": { - "tp": 1450, - "fn": 150, - "accuracy": 0.90625 - } - }, - "auroc": 0.9579713867187499 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3073, - "fn": 127, - "accuracy": 0.9603125 - }, - "0.01": { - "tp": 2940, - "fn": 260, - "accuracy": 0.91875 - } - }, - "auroc": 0.9628012532552084 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3073, - "fn": 127, - "accuracy": 0.9603125 - }, - "0.01": { - "tp": 2940, - "fn": 260, - "accuracy": 0.91875 - } - }, - "auroc": 0.9628012532552084 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1483, - "fn": 117, - "accuracy": 0.926875 - }, - "0.01": { - "tp": 1349, - "fn": 251, - "accuracy": 0.843125 - } - }, - "auroc": 0.9361846191406249 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1483, - "fn": 117, - "accuracy": 0.926875 - }, - "0.01": { - "tp": 1349, - "fn": 251, - "accuracy": 0.843125 - } - }, - "auroc": 0.9361846191406249 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1397, - "fn": 203, - "accuracy": 0.873125 - }, - "0.01": { - "tp": 1228, - "fn": 372, - "accuracy": 0.7675 - } - }, - "auroc": 0.9129304524739583 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1397, - "fn": 203, - "accuracy": 0.873125 - }, - "0.01": { - "tp": 1228, - "fn": 372, - "accuracy": 0.7675 - } - }, - "auroc": 0.9129304524739583 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2880, - "fn": 320, - "accuracy": 0.9 - }, - "0.01": { - "tp": 2577, - "fn": 623, - "accuracy": 0.8053125 - } - }, - "auroc": 0.9245575358072917 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2880, - "fn": 320, - "accuracy": 0.9 - }, - "0.01": { - "tp": 2577, - "fn": 623, - "accuracy": 0.8053125 - } - }, - "auroc": 0.9245575358072917 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9978628092447918 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9978628092447918 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - } - }, - "auroc": 0.9956561197916667 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - } - }, - "auroc": 0.9956561197916667 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3188, - "fn": 12, - "accuracy": 0.99625 - } - }, - "auroc": 0.9967594645182292 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3188, - "fn": 12, - "accuracy": 0.99625 - } - }, - "auroc": 0.9967594645182292 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9983534830729167 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9983534830729167 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1557, - "fn": 43, - "accuracy": 0.973125 - } - }, - "auroc": 0.9817611002604166 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1557, - "fn": 43, - "accuracy": 0.973125 - } - }, - "auroc": 0.9817611002604166 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3186, - "fn": 14, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 3152, - "fn": 48, - "accuracy": 0.985 - } - }, - "auroc": 0.9900572916666667 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 3186, - "fn": 14, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 3152, - "fn": 48, - "accuracy": 0.985 - } - }, - "auroc": 0.9900572916666667 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1521, - "fn": 79, - "accuracy": 0.950625 - }, - "0.01": { - "tp": 1446, - "fn": 154, - "accuracy": 0.90375 - } - }, - "auroc": 0.9655146321614583 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1521, - "fn": 79, - "accuracy": 0.950625 - }, - "0.01": { - "tp": 1446, - "fn": 154, - "accuracy": 0.90375 - } - }, - "auroc": 0.9655146321614583 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1470, - "fn": 130, - "accuracy": 0.91875 - }, - "0.01": { - "tp": 1372, - "fn": 228, - "accuracy": 0.8575 - } - }, - "auroc": 0.9468695475260416 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 1470, - "fn": 130, - "accuracy": 0.91875 - }, - "0.01": { - "tp": 1372, - "fn": 228, - "accuracy": 0.8575 - } - }, - "auroc": 0.9468695475260416 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2991, - "fn": 209, - "accuracy": 0.9346875 - }, - "0.01": { - "tp": 2818, - "fn": 382, - "accuracy": 0.880625 - } - }, - "auroc": 0.9561920898437501 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 2991, - "fn": 209, - "accuracy": 0.9346875 - }, - "0.01": { - "tp": 2818, - "fn": 382, - "accuracy": 0.880625 - } - }, - "auroc": 0.9561920898437501 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 17178, - "fn": 422, - "accuracy": 0.9760227272727273 - }, - "0.01": { - "tp": 16651, - "fn": 949, - "accuracy": 0.9460795454545454 - } - }, - "auroc": 0.9781171860203598 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 9548, - "fn": 52, - "accuracy": 0.9945833333333334 - }, - "0.01": { - "tp": 9464, - "fn": 136, - "accuracy": 0.9858333333333333 - } - }, - "auroc": 0.9910063069661459 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 26726, - "fn": 474, - "accuracy": 0.9825735294117647 - }, - "0.01": { - "tp": 26115, - "fn": 1085, - "accuracy": 0.960110294117647 - } - }, - "auroc": 0.9826662875306373 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 16678, - "fn": 922, - "accuracy": 0.9476136363636364 - }, - "0.01": { - "tp": 15655, - "fn": 1945, - "accuracy": 0.8894886363636364 - } - }, - "auroc": 0.9576630726207387 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 9361, - "fn": 239, - "accuracy": 0.9751041666666667 - }, - "0.01": { - "tp": 9030, - "fn": 570, - "accuracy": 0.940625 - } - }, - "auroc": 0.9678104546440973 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 26039, - "fn": 1161, - "accuracy": 0.9573161764705882 - }, - "0.01": { - "tp": 24685, - "fn": 2515, - "accuracy": 0.9075367647058824 - } - }, - "auroc": 0.9612445015701594 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 33856, - "fn": 1344, - "accuracy": 0.9618181818181818 - }, - "0.01": { - "tp": 32306, - "fn": 2894, - "accuracy": 0.9177840909090909 - } - }, - "auroc": 0.9678901293205493 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 18909, - "fn": 291, - "accuracy": 0.98484375 - }, - "0.01": { - "tp": 18494, - "fn": 706, - "accuracy": 0.9632291666666667 - } - }, - "auroc": 0.9794083808051215 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "insert_paragraphs", - "accuracy": { - "0.05": { - "tp": 52765, - "fn": 1635, - "accuracy": 0.9699448529411765 - }, - "0.01": { - "tp": 50800, - "fn": 3600, - "accuracy": 0.9338235294117647 - } - }, - "auroc": 0.9719553945503984 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1581, - "fn": 19, - "accuracy": 0.988125 - } - }, - "auroc": 0.9497156575520833 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1570, - "fn": 30, - "accuracy": 0.98125 - } - }, - "auroc": 0.93613056640625 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3151, - "fn": 49, - "accuracy": 0.9846875 - } - }, - "auroc": 0.9429231119791667 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1579, - "fn": 21, - "accuracy": 0.986875 - } - }, - "auroc": 0.9495599772135417 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1573, - "fn": 27, - "accuracy": 0.983125 - } - }, - "auroc": 0.938601904296875 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3199, - "fn": 1, - "accuracy": 0.9996875 - }, - "0.01": { - "tp": 3152, - "fn": 48, - "accuracy": 0.985 - } - }, - "auroc": 0.9440809407552082 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3160, - "fn": 40, - "accuracy": 0.9875 - } - }, - "auroc": 0.9496378173828125 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3199, - "fn": 1, - "accuracy": 0.9996875 - }, - "0.01": { - "tp": 3143, - "fn": 57, - "accuracy": 0.9821875 - } - }, - "auroc": 0.9373662353515625 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 6399, - "fn": 1, - "accuracy": 0.99984375 - }, - "0.01": { - "tp": 6303, - "fn": 97, - "accuracy": 0.98484375 - } - }, - "auroc": 0.9435020263671875 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1568, - "fn": 32, - "accuracy": 0.98 - }, - "0.01": { - "tp": 1510, - "fn": 90, - "accuracy": 0.94375 - } - }, - "auroc": 0.9803754882812501 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1581, - "fn": 19, - "accuracy": 0.988125 - }, - "0.01": { - "tp": 1570, - "fn": 30, - "accuracy": 0.98125 - } - }, - "auroc": 0.9804583821614583 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3149, - "fn": 51, - "accuracy": 0.9840625 - }, - "0.01": { - "tp": 3080, - "fn": 120, - "accuracy": 0.9625 - } - }, - "auroc": 0.9804169352213542 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - } - }, - "auroc": 0.977751708984375 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1546, - "fn": 54, - "accuracy": 0.96625 - }, - "0.01": { - "tp": 1505, - "fn": 95, - "accuracy": 0.940625 - } - }, - "auroc": 0.9719453776041667 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3142, - "fn": 58, - "accuracy": 0.981875 - }, - "0.01": { - "tp": 3097, - "fn": 103, - "accuracy": 0.9678125 - } - }, - "auroc": 0.9748485432942708 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3164, - "fn": 36, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 3102, - "fn": 98, - "accuracy": 0.969375 - } - }, - "auroc": 0.9790635986328126 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3127, - "fn": 73, - "accuracy": 0.9771875 - }, - "0.01": { - "tp": 3075, - "fn": 125, - "accuracy": 0.9609375 - } - }, - "auroc": 0.9762018798828125 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 6291, - "fn": 109, - "accuracy": 0.98296875 - }, - "0.01": { - "tp": 6177, - "fn": 223, - "accuracy": 0.96515625 - } - }, - "auroc": 0.9776327392578125 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - } - }, - "auroc": 0.96244501953125 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 1586, - "fn": 14, - "accuracy": 0.99125 - } - }, - "auroc": 0.9586985514322917 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3186, - "fn": 14, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 3175, - "fn": 25, - "accuracy": 0.9921875 - } - }, - "auroc": 0.9605717854817708 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1581, - "fn": 19, - "accuracy": 0.988125 - } - }, - "auroc": 0.9591965332031249 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 1577, - "fn": 23, - "accuracy": 0.985625 - } - }, - "auroc": 0.9659574381510416 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3187, - "fn": 13, - "accuracy": 0.9959375 - }, - "0.01": { - "tp": 3158, - "fn": 42, - "accuracy": 0.986875 - } - }, - "auroc": 0.9625769856770834 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3192, - "fn": 8, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 3170, - "fn": 30, - "accuracy": 0.990625 - } - }, - "auroc": 0.9608207763671875 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3181, - "fn": 19, - "accuracy": 0.9940625 - }, - "0.01": { - "tp": 3163, - "fn": 37, - "accuracy": 0.9884375 - } - }, - "auroc": 0.9623279947916668 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 6373, - "fn": 27, - "accuracy": 0.99578125 - }, - "0.01": { - "tp": 6333, - "fn": 67, - "accuracy": 0.98953125 - } - }, - "auroc": 0.9615743855794272 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1514, - "fn": 86, - "accuracy": 0.94625 - }, - "0.01": { - "tp": 1388, - "fn": 212, - "accuracy": 0.8675 - } - }, - "auroc": 0.9556012858072918 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 1581, - "fn": 19, - "accuracy": 0.988125 - } - }, - "auroc": 0.9853427734375 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3109, - "fn": 91, - "accuracy": 0.9715625 - }, - "0.01": { - "tp": 2969, - "fn": 231, - "accuracy": 0.9278125 - } - }, - "auroc": 0.9704720296223959 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 1573, - "fn": 27, - "accuracy": 0.983125 - } - }, - "auroc": 0.9828428385416667 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1575, - "fn": 25, - "accuracy": 0.984375 - }, - "0.01": { - "tp": 1561, - "fn": 39, - "accuracy": 0.975625 - } - }, - "auroc": 0.9751118001302084 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3166, - "fn": 34, - "accuracy": 0.989375 - }, - "0.01": { - "tp": 3134, - "fn": 66, - "accuracy": 0.979375 - } - }, - "auroc": 0.9789773193359376 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3105, - "fn": 95, - "accuracy": 0.9703125 - }, - "0.01": { - "tp": 2961, - "fn": 239, - "accuracy": 0.9253125 - } - }, - "auroc": 0.9692220621744791 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3170, - "fn": 30, - "accuracy": 0.990625 - }, - "0.01": { - "tp": 3142, - "fn": 58, - "accuracy": 0.981875 - } - }, - "auroc": 0.9802272867838541 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 6275, - "fn": 125, - "accuracy": 0.98046875 - }, - "0.01": { - "tp": 6103, - "fn": 297, - "accuracy": 0.95359375 - } - }, - "auroc": 0.9747246744791667 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1564, - "fn": 36, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 1510, - "fn": 90, - "accuracy": 0.94375 - } - }, - "auroc": 0.9813230794270833 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - } - }, - "auroc": 0.9759094889322917 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3164, - "fn": 36, - "accuracy": 0.98875 - }, - "0.01": { - "tp": 3102, - "fn": 98, - "accuracy": 0.969375 - } - }, - "auroc": 0.9786162841796875 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1593, - "fn": 7, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 1577, - "fn": 23, - "accuracy": 0.985625 - } - }, - "auroc": 0.9753461751302084 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1585, - "fn": 15, - "accuracy": 0.990625 - } - }, - "auroc": 0.972777783203125 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3187, - "fn": 13, - "accuracy": 0.9959375 - }, - "0.01": { - "tp": 3162, - "fn": 38, - "accuracy": 0.988125 - } - }, - "auroc": 0.9740619791666667 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3157, - "fn": 43, - "accuracy": 0.9865625 - }, - "0.01": { - "tp": 3087, - "fn": 113, - "accuracy": 0.9646875 - } - }, - "auroc": 0.9783346272786458 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3194, - "fn": 6, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 3177, - "fn": 23, - "accuracy": 0.9928125 - } - }, - "auroc": 0.9743436360677084 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 6351, - "fn": 49, - "accuracy": 0.99234375 - }, - "0.01": { - "tp": 6264, - "fn": 136, - "accuracy": 0.97875 - } - }, - "auroc": 0.9763391316731771 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1573, - "fn": 27, - "accuracy": 0.983125 - } - }, - "auroc": 0.962117138671875 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1598, - "fn": 2, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 1577, - "fn": 23, - "accuracy": 0.985625 - } - }, - "auroc": 0.9464330403645833 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3194, - "fn": 6, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 3150, - "fn": 50, - "accuracy": 0.984375 - } - }, - "auroc": 0.9542750895182291 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1572, - "fn": 28, - "accuracy": 0.9825 - } - }, - "auroc": 0.95548974609375 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - } - }, - "auroc": 0.9513683919270833 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3188, - "fn": 12, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 3148, - "fn": 52, - "accuracy": 0.98375 - } - }, - "auroc": 0.9534290690104166 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3145, - "fn": 55, - "accuracy": 0.9828125 - } - }, - "auroc": 0.9588034423828125 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3192, - "fn": 8, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 3153, - "fn": 47, - "accuracy": 0.9853125 - } - }, - "auroc": 0.9489007161458334 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 6382, - "fn": 18, - "accuracy": 0.9971875 - }, - "0.01": { - "tp": 6298, - "fn": 102, - "accuracy": 0.9840625 - } - }, - "auroc": 0.9538520792643229 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1573, - "fn": 27, - "accuracy": 0.983125 - }, - "0.01": { - "tp": 1544, - "fn": 56, - "accuracy": 0.965 - } - }, - "auroc": 0.9760803873697916 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1573, - "fn": 27, - "accuracy": 0.983125 - }, - "0.01": { - "tp": 1544, - "fn": 56, - "accuracy": 0.965 - } - }, - "auroc": 0.9760803873697916 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1571, - "fn": 29, - "accuracy": 0.981875 - }, - "0.01": { - "tp": 1537, - "fn": 63, - "accuracy": 0.960625 - } - }, - "auroc": 0.9721631998697917 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1571, - "fn": 29, - "accuracy": 0.981875 - }, - "0.01": { - "tp": 1537, - "fn": 63, - "accuracy": 0.960625 - } - }, - "auroc": 0.9721631998697917 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3144, - "fn": 56, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 3081, - "fn": 119, - "accuracy": 0.9628125 - } - }, - "auroc": 0.9741217936197917 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3144, - "fn": 56, - "accuracy": 0.9825 - }, - "0.01": { - "tp": 3081, - "fn": 119, - "accuracy": 0.9628125 - } - }, - "auroc": 0.9741217936197917 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1586, - "fn": 14, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - } - }, - "auroc": 0.9693377604166666 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1586, - "fn": 14, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - } - }, - "auroc": 0.9693377604166666 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1585, - "fn": 15, - "accuracy": 0.990625 - }, - "0.01": { - "tp": 1568, - "fn": 32, - "accuracy": 0.98 - } - }, - "auroc": 0.9621617350260416 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1585, - "fn": 15, - "accuracy": 0.990625 - }, - "0.01": { - "tp": 1568, - "fn": 32, - "accuracy": 0.98 - } - }, - "auroc": 0.9621617350260416 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3171, - "fn": 29, - "accuracy": 0.9909375 - }, - "0.01": { - "tp": 3144, - "fn": 56, - "accuracy": 0.9825 - } - }, - "auroc": 0.9657497477213541 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3171, - "fn": 29, - "accuracy": 0.9909375 - }, - "0.01": { - "tp": 3144, - "fn": 56, - "accuracy": 0.9825 - } - }, - "auroc": 0.9657497477213541 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1531, - "fn": 69, - "accuracy": 0.956875 - } - }, - "auroc": 0.9379755045572916 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1531, - "fn": 69, - "accuracy": 0.956875 - } - }, - "auroc": 0.9379755045572916 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1537, - "fn": 63, - "accuracy": 0.960625 - } - }, - "auroc": 0.9357070149739583 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1537, - "fn": 63, - "accuracy": 0.960625 - } - }, - "auroc": 0.9357070149739583 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3198, - "fn": 2, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 3068, - "fn": 132, - "accuracy": 0.95875 - } - }, - "auroc": 0.936841259765625 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3198, - "fn": 2, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 3068, - "fn": 132, - "accuracy": 0.95875 - } - }, - "auroc": 0.936841259765625 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1542, - "fn": 58, - "accuracy": 0.96375 - } - }, - "auroc": 0.9404706054687499 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1542, - "fn": 58, - "accuracy": 0.96375 - } - }, - "auroc": 0.9404706054687499 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - } - }, - "auroc": 0.9451950683593751 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - } - }, - "auroc": 0.9451950683593751 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3198, - "fn": 2, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 3134, - "fn": 66, - "accuracy": 0.979375 - } - }, - "auroc": 0.9428328369140626 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3198, - "fn": 2, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 3134, - "fn": 66, - "accuracy": 0.979375 - } - }, - "auroc": 0.9428328369140626 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 1579, - "fn": 21, - "accuracy": 0.986875 - } - }, - "auroc": 0.9679061360677084 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 1579, - "fn": 21, - "accuracy": 0.986875 - } - }, - "auroc": 0.9679061360677084 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1575, - "fn": 25, - "accuracy": 0.984375 - }, - "0.01": { - "tp": 1550, - "fn": 50, - "accuracy": 0.96875 - } - }, - "auroc": 0.9615626139322917 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 1575, - "fn": 25, - "accuracy": 0.984375 - }, - "0.01": { - "tp": 1550, - "fn": 50, - "accuracy": 0.96875 - } - }, - "auroc": 0.9615626139322917 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3165, - "fn": 35, - "accuracy": 0.9890625 - }, - "0.01": { - "tp": 3129, - "fn": 71, - "accuracy": 0.9778125 - } - }, - "auroc": 0.964734375 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 3165, - "fn": 35, - "accuracy": 0.9890625 - }, - "0.01": { - "tp": 3129, - "fn": 71, - "accuracy": 0.9778125 - } - }, - "auroc": 0.964734375 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 17384, - "fn": 216, - "accuracy": 0.9877272727272727 - }, - "0.01": { - "tp": 16923, - "fn": 677, - "accuracy": 0.9615340909090909 - } - }, - "auroc": 0.9621225511955492 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 9565, - "fn": 35, - "accuracy": 0.9963541666666667 - }, - "0.01": { - "tp": 9476, - "fn": 124, - "accuracy": 0.9870833333333333 - } - }, - "auroc": 0.9638288004557292 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 26949, - "fn": 251, - "accuracy": 0.9907720588235294 - }, - "0.01": { - "tp": 26399, - "fn": 801, - "accuracy": 0.9705514705882353 - } - }, - "auroc": 0.9627247568167892 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 17500, - "fn": 100, - "accuracy": 0.9943181818181818 - }, - "0.01": { - "tp": 17258, - "fn": 342, - "accuracy": 0.9805681818181818 - } - }, - "auroc": 0.9615433283025568 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 9498, - "fn": 102, - "accuracy": 0.989375 - }, - "0.01": { - "tp": 9377, - "fn": 223, - "accuracy": 0.9767708333333334 - } - }, - "auroc": 0.9626271158854167 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 26998, - "fn": 202, - "accuracy": 0.9925735294117647 - }, - "0.01": { - "tp": 26635, - "fn": 565, - "accuracy": 0.9792279411764706 - } - }, - "auroc": 0.9619258415670955 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 34884, - "fn": 316, - "accuracy": 0.9910227272727272 - }, - "0.01": { - "tp": 34181, - "fn": 1019, - "accuracy": 0.9710511363636364 - } - }, - "auroc": 0.961832939749053 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 19063, - "fn": 137, - "accuracy": 0.9928645833333334 - }, - "0.01": { - "tp": 18853, - "fn": 347, - "accuracy": 0.9819270833333333 - } - }, - "auroc": 0.9632279581705729 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "homoglyph", - "accuracy": { - "0.05": { - "tp": 53947, - "fn": 453, - "accuracy": 0.9916727941176471 - }, - "0.01": { - "tp": 53034, - "fn": 1366, - "accuracy": 0.9748897058823529 - } - }, - "auroc": 0.9623252991919424 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1598, - "fn": 2, - "accuracy": 0.99875 - } - }, - "auroc": 0.9866437662760417 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9862053059895833 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3193, - "fn": 7, - "accuracy": 0.9978125 - } - }, - "auroc": 0.9864245361328124 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1593, - "fn": 7, - "accuracy": 0.995625 - } - }, - "auroc": 0.9851371256510417 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1598, - "fn": 2, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 1583, - "fn": 17, - "accuracy": 0.989375 - } - }, - "auroc": 0.980878515625 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3198, - "fn": 2, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 3176, - "fn": 24, - "accuracy": 0.9925 - } - }, - "auroc": 0.9830078206380208 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3191, - "fn": 9, - "accuracy": 0.9971875 - } - }, - "auroc": 0.9858904459635417 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3198, - "fn": 2, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 3178, - "fn": 22, - "accuracy": 0.993125 - } - }, - "auroc": 0.9835419108072917 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 6398, - "fn": 2, - "accuracy": 0.9996875 - }, - "0.01": { - "tp": 6369, - "fn": 31, - "accuracy": 0.99515625 - } - }, - "auroc": 0.9847161783854166 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1562, - "fn": 38, - "accuracy": 0.97625 - }, - "0.01": { - "tp": 1466, - "fn": 134, - "accuracy": 0.91625 - } - }, - "auroc": 0.9557456054687501 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - }, - "0.01": { - "tp": 1554, - "fn": 46, - "accuracy": 0.97125 - } - }, - "auroc": 0.986875732421875 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3138, - "fn": 62, - "accuracy": 0.980625 - }, - "0.01": { - "tp": 3020, - "fn": 180, - "accuracy": 0.94375 - } - }, - "auroc": 0.9713106689453125 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1549, - "fn": 51, - "accuracy": 0.968125 - }, - "0.01": { - "tp": 1398, - "fn": 202, - "accuracy": 0.87375 - } - }, - "auroc": 0.9229142089843749 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1534, - "fn": 66, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 1483, - "fn": 117, - "accuracy": 0.926875 - } - }, - "auroc": 0.9657796386718751 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3083, - "fn": 117, - "accuracy": 0.9634375 - }, - "0.01": { - "tp": 2881, - "fn": 319, - "accuracy": 0.9003125 - } - }, - "auroc": 0.944346923828125 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3111, - "fn": 89, - "accuracy": 0.9721875 - }, - "0.01": { - "tp": 2864, - "fn": 336, - "accuracy": 0.895 - } - }, - "auroc": 0.9393299072265625 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3110, - "fn": 90, - "accuracy": 0.971875 - }, - "0.01": { - "tp": 3037, - "fn": 163, - "accuracy": 0.9490625 - } - }, - "auroc": 0.976327685546875 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 6221, - "fn": 179, - "accuracy": 0.97203125 - }, - "0.01": { - "tp": 5901, - "fn": 499, - "accuracy": 0.92203125 - } - }, - "auroc": 0.9578287963867187 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1577, - "fn": 23, - "accuracy": 0.985625 - } - }, - "auroc": 0.98349169921875 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 1572, - "fn": 28, - "accuracy": 0.9825 - } - }, - "auroc": 0.9836621907552083 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3179, - "fn": 21, - "accuracy": 0.9934375 - }, - "0.01": { - "tp": 3149, - "fn": 51, - "accuracy": 0.9840625 - } - }, - "auroc": 0.9835769449869791 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1564, - "fn": 36, - "accuracy": 0.9775 - } - }, - "auroc": 0.9793331705729167 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - }, - "0.01": { - "tp": 1527, - "fn": 73, - "accuracy": 0.954375 - } - }, - "auroc": 0.9686875325520834 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3173, - "fn": 27, - "accuracy": 0.9915625 - }, - "0.01": { - "tp": 3091, - "fn": 109, - "accuracy": 0.9659375 - } - }, - "auroc": 0.9740103515625 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3178, - "fn": 22, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 3141, - "fn": 59, - "accuracy": 0.9815625 - } - }, - "auroc": 0.9814124348958333 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3174, - "fn": 26, - "accuracy": 0.991875 - }, - "0.01": { - "tp": 3099, - "fn": 101, - "accuracy": 0.9684375 - } - }, - "auroc": 0.9761748616536459 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 6352, - "fn": 48, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 6240, - "fn": 160, - "accuracy": 0.975 - } - }, - "auroc": 0.9787936482747396 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1512, - "fn": 88, - "accuracy": 0.945 - }, - "0.01": { - "tp": 1342, - "fn": 258, - "accuracy": 0.83875 - } - }, - "auroc": 0.937257080078125 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 1569, - "fn": 31, - "accuracy": 0.980625 - } - }, - "auroc": 0.9812041341145833 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3107, - "fn": 93, - "accuracy": 0.9709375 - }, - "0.01": { - "tp": 2911, - "fn": 289, - "accuracy": 0.9096875 - } - }, - "auroc": 0.9592306070963541 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1565, - "fn": 35, - "accuracy": 0.978125 - }, - "0.01": { - "tp": 1403, - "fn": 197, - "accuracy": 0.876875 - } - }, - "auroc": 0.9213321940104168 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1560, - "fn": 40, - "accuracy": 0.975 - }, - "0.01": { - "tp": 1514, - "fn": 86, - "accuracy": 0.94625 - } - }, - "auroc": 0.9639603678385416 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3125, - "fn": 75, - "accuracy": 0.9765625 - }, - "0.01": { - "tp": 2917, - "fn": 283, - "accuracy": 0.9115625 - } - }, - "auroc": 0.9426462809244791 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3077, - "fn": 123, - "accuracy": 0.9615625 - }, - "0.01": { - "tp": 2745, - "fn": 455, - "accuracy": 0.8578125 - } - }, - "auroc": 0.9292946370442708 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3155, - "fn": 45, - "accuracy": 0.9859375 - }, - "0.01": { - "tp": 3083, - "fn": 117, - "accuracy": 0.9634375 - } - }, - "auroc": 0.9725822509765625 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 6232, - "fn": 168, - "accuracy": 0.97375 - }, - "0.01": { - "tp": 5828, - "fn": 572, - "accuracy": 0.910625 - } - }, - "auroc": 0.9509384440104167 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1563, - "fn": 37, - "accuracy": 0.976875 - }, - "0.01": { - "tp": 1484, - "fn": 116, - "accuracy": 0.9275 - } - }, - "auroc": 0.9671744303385417 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1573, - "fn": 27, - "accuracy": 0.983125 - } - }, - "auroc": 0.9817627441406251 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3157, - "fn": 43, - "accuracy": 0.9865625 - }, - "0.01": { - "tp": 3057, - "fn": 143, - "accuracy": 0.9553125 - } - }, - "auroc": 0.9744685872395833 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1499, - "fn": 101, - "accuracy": 0.936875 - }, - "0.01": { - "tp": 1184, - "fn": 416, - "accuracy": 0.74 - } - }, - "auroc": 0.8972121256510417 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1547, - "fn": 53, - "accuracy": 0.966875 - }, - "0.01": { - "tp": 1464, - "fn": 136, - "accuracy": 0.915 - } - }, - "auroc": 0.9547926595052083 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3046, - "fn": 154, - "accuracy": 0.951875 - }, - "0.01": { - "tp": 2648, - "fn": 552, - "accuracy": 0.8275 - } - }, - "auroc": 0.9260023925781251 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3062, - "fn": 138, - "accuracy": 0.956875 - }, - "0.01": { - "tp": 2668, - "fn": 532, - "accuracy": 0.83375 - } - }, - "auroc": 0.9321932779947917 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3141, - "fn": 59, - "accuracy": 0.9815625 - }, - "0.01": { - "tp": 3037, - "fn": 163, - "accuracy": 0.9490625 - } - }, - "auroc": 0.9682777018229167 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 6203, - "fn": 197, - "accuracy": 0.96921875 - }, - "0.01": { - "tp": 5705, - "fn": 695, - "accuracy": 0.89140625 - } - }, - "auroc": 0.9502354899088543 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - } - }, - "auroc": 0.9859532389322916 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - } - }, - "auroc": 0.987436767578125 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3192, - "fn": 8, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 3183, - "fn": 17, - "accuracy": 0.9946875 - } - }, - "auroc": 0.9866950032552083 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1588, - "fn": 12, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 1578, - "fn": 22, - "accuracy": 0.98625 - } - }, - "auroc": 0.9793545735677084 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1578, - "fn": 22, - "accuracy": 0.98625 - }, - "0.01": { - "tp": 1537, - "fn": 63, - "accuracy": 0.960625 - } - }, - "auroc": 0.9717302408854167 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3166, - "fn": 34, - "accuracy": 0.989375 - }, - "0.01": { - "tp": 3115, - "fn": 85, - "accuracy": 0.9734375 - } - }, - "auroc": 0.9755424072265625 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3184, - "fn": 16, - "accuracy": 0.995 - }, - "0.01": { - "tp": 3169, - "fn": 31, - "accuracy": 0.9903125 - } - }, - "auroc": 0.9826539062499999 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3174, - "fn": 26, - "accuracy": 0.991875 - }, - "0.01": { - "tp": 3129, - "fn": 71, - "accuracy": 0.9778125 - } - }, - "auroc": 0.9795835042317708 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 6358, - "fn": 42, - "accuracy": 0.9934375 - }, - "0.01": { - "tp": 6298, - "fn": 102, - "accuracy": 0.9840625 - } - }, - "auroc": 0.9811187052408854 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1534, - "fn": 66, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 1434, - "fn": 166, - "accuracy": 0.89625 - } - }, - "auroc": 0.9374627766927084 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1534, - "fn": 66, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 1434, - "fn": 166, - "accuracy": 0.89625 - } - }, - "auroc": 0.9374627766927084 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1515, - "fn": 85, - "accuracy": 0.946875 - }, - "0.01": { - "tp": 1375, - "fn": 225, - "accuracy": 0.859375 - } - }, - "auroc": 0.9263558756510417 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1515, - "fn": 85, - "accuracy": 0.946875 - }, - "0.01": { - "tp": 1375, - "fn": 225, - "accuracy": 0.859375 - } - }, - "auroc": 0.9263558756510417 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3049, - "fn": 151, - "accuracy": 0.9528125 - }, - "0.01": { - "tp": 2809, - "fn": 391, - "accuracy": 0.8778125 - } - }, - "auroc": 0.9319093261718749 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3049, - "fn": 151, - "accuracy": 0.9528125 - }, - "0.01": { - "tp": 2809, - "fn": 391, - "accuracy": 0.8778125 - } - }, - "auroc": 0.9319093261718749 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1494, - "fn": 106, - "accuracy": 0.93375 - }, - "0.01": { - "tp": 1307, - "fn": 293, - "accuracy": 0.816875 - } - }, - "auroc": 0.9075196777343749 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1494, - "fn": 106, - "accuracy": 0.93375 - }, - "0.01": { - "tp": 1307, - "fn": 293, - "accuracy": 0.816875 - } - }, - "auroc": 0.9075196777343749 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1453, - "fn": 147, - "accuracy": 0.908125 - }, - "0.01": { - "tp": 1161, - "fn": 439, - "accuracy": 0.725625 - } - }, - "auroc": 0.8893345214843751 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1453, - "fn": 147, - "accuracy": 0.908125 - }, - "0.01": { - "tp": 1161, - "fn": 439, - "accuracy": 0.725625 - } - }, - "auroc": 0.8893345214843751 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2947, - "fn": 253, - "accuracy": 0.9209375 - }, - "0.01": { - "tp": 2468, - "fn": 732, - "accuracy": 0.77125 - } - }, - "auroc": 0.898427099609375 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 2947, - "fn": 253, - "accuracy": 0.9209375 - }, - "0.01": { - "tp": 2468, - "fn": 732, - "accuracy": 0.77125 - } - }, - "auroc": 0.898427099609375 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1593, - "fn": 7, - "accuracy": 0.995625 - } - }, - "auroc": 0.9874363118489584 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1593, - "fn": 7, - "accuracy": 0.995625 - } - }, - "auroc": 0.9874363118489584 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1588, - "fn": 12, - "accuracy": 0.9925 - } - }, - "auroc": 0.9838117838541667 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1588, - "fn": 12, - "accuracy": 0.9925 - } - }, - "auroc": 0.9838117838541667 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3181, - "fn": 19, - "accuracy": 0.9940625 - } - }, - "auroc": 0.9856240478515625 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3181, - "fn": 19, - "accuracy": 0.9940625 - } - }, - "auroc": 0.9856240478515625 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - } - }, - "auroc": 0.9839721516927084 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1592, - "fn": 8, - "accuracy": 0.995 - } - }, - "auroc": 0.9839721516927084 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 1506, - "fn": 94, - "accuracy": 0.94125 - } - }, - "auroc": 0.9553196777343751 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 1506, - "fn": 94, - "accuracy": 0.94125 - } - }, - "auroc": 0.9553196777343751 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3191, - "fn": 9, - "accuracy": 0.9971875 - }, - "0.01": { - "tp": 3098, - "fn": 102, - "accuracy": 0.968125 - } - }, - "auroc": 0.9696459147135418 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3191, - "fn": 9, - "accuracy": 0.9971875 - }, - "0.01": { - "tp": 3098, - "fn": 102, - "accuracy": 0.968125 - } - }, - "auroc": 0.9696459147135418 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1539, - "fn": 61, - "accuracy": 0.961875 - }, - "0.01": { - "tp": 1423, - "fn": 177, - "accuracy": 0.889375 - } - }, - "auroc": 0.9464276692708333 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1539, - "fn": 61, - "accuracy": 0.961875 - }, - "0.01": { - "tp": 1423, - "fn": 177, - "accuracy": 0.889375 - } - }, - "auroc": 0.9464276692708333 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1492, - "fn": 108, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 1331, - "fn": 269, - "accuracy": 0.831875 - } - }, - "auroc": 0.9281087076822916 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 1492, - "fn": 108, - "accuracy": 0.9325 - }, - "0.01": { - "tp": 1331, - "fn": 269, - "accuracy": 0.831875 - } - }, - "auroc": 0.9281087076822916 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3031, - "fn": 169, - "accuracy": 0.9471875 - }, - "0.01": { - "tp": 2754, - "fn": 446, - "accuracy": 0.860625 - } - }, - "auroc": 0.9372681884765626 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 3031, - "fn": 169, - "accuracy": 0.9471875 - }, - "0.01": { - "tp": 2754, - "fn": 446, - "accuracy": 0.860625 - } - }, - "auroc": 0.9372681884765626 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 17181, - "fn": 419, - "accuracy": 0.9761931818181818 - }, - "0.01": { - "tp": 16407, - "fn": 1193, - "accuracy": 0.9322159090909091 - } - }, - "auroc": 0.9617349461410986 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 9551, - "fn": 49, - "accuracy": 0.9948958333333333 - }, - "0.01": { - "tp": 9455, - "fn": 145, - "accuracy": 0.9848958333333333 - } - }, - "auroc": 0.9845244791666666 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 26732, - "fn": 468, - "accuracy": 0.9827941176470588 - }, - "0.01": { - "tp": 25862, - "fn": 1338, - "accuracy": 0.9508088235294118 - } - }, - "auroc": 0.9697783107383579 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 17039, - "fn": 561, - "accuracy": 0.968125 - }, - "0.01": { - "tp": 15681, - "fn": 1919, - "accuracy": 0.8909659090909091 - } - }, - "auroc": 0.9425649058948863 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 9401, - "fn": 199, - "accuracy": 0.9792708333333333 - }, - "0.01": { - "tp": 9108, - "fn": 492, - "accuracy": 0.94875 - } - }, - "auroc": 0.9676381591796874 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 26440, - "fn": 760, - "accuracy": 0.9720588235294118 - }, - "0.01": { - "tp": 24789, - "fn": 2411, - "accuracy": 0.9113602941176471 - } - }, - "auroc": 0.9514142894071691 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 34220, - "fn": 980, - "accuracy": 0.9721590909090909 - }, - "0.01": { - "tp": 32088, - "fn": 3112, - "accuracy": 0.9115909090909091 - } - }, - "auroc": 0.9521499260179924 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 18952, - "fn": 248, - "accuracy": 0.9870833333333333 - }, - "0.01": { - "tp": 18563, - "fn": 637, - "accuracy": 0.9668229166666666 - } - }, - "auroc": 0.9760813191731771 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "article_deletion", - "accuracy": { - "0.05": { - "tp": 53172, - "fn": 1228, - "accuracy": 0.9774264705882353 - }, - "0.01": { - "tp": 50651, - "fn": 3749, - "accuracy": 0.9310845588235294 - } - }, - "auroc": 0.9605963000727634 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - } - }, - "auroc": 0.9965575358072917 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - } - }, - "auroc": 0.9950832031250001 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3198, - "fn": 2, - "accuracy": 0.999375 - } - }, - "auroc": 0.9958203694661458 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - } - }, - "auroc": 0.9956486490885417 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1598, - "fn": 2, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 1583, - "fn": 17, - "accuracy": 0.989375 - } - }, - "auroc": 0.990886865234375 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3198, - "fn": 2, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 3180, - "fn": 20, - "accuracy": 0.99375 - } - }, - "auroc": 0.9932677571614583 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3196, - "fn": 4, - "accuracy": 0.99875 - } - }, - "auroc": 0.9961030924479166 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3198, - "fn": 2, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 3182, - "fn": 18, - "accuracy": 0.994375 - } - }, - "auroc": 0.9929850341796875 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 6398, - "fn": 2, - "accuracy": 0.9996875 - }, - "0.01": { - "tp": 6378, - "fn": 22, - "accuracy": 0.9965625 - } - }, - "auroc": 0.9945440633138021 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1564, - "fn": 36, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 1495, - "fn": 105, - "accuracy": 0.934375 - } - }, - "auroc": 0.9728748209635417 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - }, - "0.01": { - "tp": 1560, - "fn": 40, - "accuracy": 0.975 - } - }, - "auroc": 0.9878343424479167 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3140, - "fn": 60, - "accuracy": 0.98125 - }, - "0.01": { - "tp": 3055, - "fn": 145, - "accuracy": 0.9546875 - } - }, - "auroc": 0.9803545817057291 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1498, - "fn": 102, - "accuracy": 0.93625 - }, - "0.01": { - "tp": 1371, - "fn": 229, - "accuracy": 0.856875 - } - }, - "auroc": 0.9419869954427084 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1535, - "fn": 65, - "accuracy": 0.959375 - }, - "0.01": { - "tp": 1483, - "fn": 117, - "accuracy": 0.926875 - } - }, - "auroc": 0.9663536132812499 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3033, - "fn": 167, - "accuracy": 0.9478125 - }, - "0.01": { - "tp": 2854, - "fn": 346, - "accuracy": 0.891875 - } - }, - "auroc": 0.9541703043619792 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3062, - "fn": 138, - "accuracy": 0.956875 - }, - "0.01": { - "tp": 2866, - "fn": 334, - "accuracy": 0.895625 - } - }, - "auroc": 0.9574309082031249 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3111, - "fn": 89, - "accuracy": 0.9721875 - }, - "0.01": { - "tp": 3043, - "fn": 157, - "accuracy": 0.9509375 - } - }, - "auroc": 0.9770939778645833 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 6173, - "fn": 227, - "accuracy": 0.96453125 - }, - "0.01": { - "tp": 5909, - "fn": 491, - "accuracy": 0.92328125 - } - }, - "auroc": 0.9672624430338542 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - } - }, - "auroc": 0.9925470377604166 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 1578, - "fn": 22, - "accuracy": 0.98625 - } - }, - "auroc": 0.9870500162760416 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3180, - "fn": 20, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 3162, - "fn": 38, - "accuracy": 0.988125 - } - }, - "auroc": 0.9897985270182292 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1586, - "fn": 14, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - } - }, - "auroc": 0.9903850260416667 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1583, - "fn": 17, - "accuracy": 0.989375 - }, - "0.01": { - "tp": 1524, - "fn": 76, - "accuracy": 0.9525 - } - }, - "auroc": 0.970997314453125 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3169, - "fn": 31, - "accuracy": 0.9903125 - }, - "0.01": { - "tp": 3100, - "fn": 100, - "accuracy": 0.96875 - } - }, - "auroc": 0.9806911702473957 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3176, - "fn": 24, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 3160, - "fn": 40, - "accuracy": 0.9875 - } - }, - "auroc": 0.9914660319010417 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3173, - "fn": 27, - "accuracy": 0.9915625 - }, - "0.01": { - "tp": 3102, - "fn": 98, - "accuracy": 0.969375 - } - }, - "auroc": 0.9790236653645833 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 6349, - "fn": 51, - "accuracy": 0.99203125 - }, - "0.01": { - "tp": 6262, - "fn": 138, - "accuracy": 0.9784375 - } - }, - "auroc": 0.9852448486328125 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1517, - "fn": 83, - "accuracy": 0.948125 - }, - "0.01": { - "tp": 1402, - "fn": 198, - "accuracy": 0.87625 - } - }, - "auroc": 0.9521694986979167 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1573, - "fn": 27, - "accuracy": 0.983125 - } - }, - "auroc": 0.99201875 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3111, - "fn": 89, - "accuracy": 0.9721875 - }, - "0.01": { - "tp": 2975, - "fn": 225, - "accuracy": 0.9296875 - } - }, - "auroc": 0.9720941243489583 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1530, - "fn": 70, - "accuracy": 0.95625 - }, - "0.01": { - "tp": 1368, - "fn": 232, - "accuracy": 0.855 - } - }, - "auroc": 0.9459575032552083 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1555, - "fn": 45, - "accuracy": 0.971875 - }, - "0.01": { - "tp": 1507, - "fn": 93, - "accuracy": 0.941875 - } - }, - "auroc": 0.9619657063802083 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3085, - "fn": 115, - "accuracy": 0.9640625 - }, - "0.01": { - "tp": 2875, - "fn": 325, - "accuracy": 0.8984375 - } - }, - "auroc": 0.9539616048177083 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3047, - "fn": 153, - "accuracy": 0.9521875 - }, - "0.01": { - "tp": 2770, - "fn": 430, - "accuracy": 0.865625 - } - }, - "auroc": 0.9490635009765624 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3149, - "fn": 51, - "accuracy": 0.9840625 - }, - "0.01": { - "tp": 3080, - "fn": 120, - "accuracy": 0.9625 - } - }, - "auroc": 0.9769922281901041 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 6196, - "fn": 204, - "accuracy": 0.968125 - }, - "0.01": { - "tp": 5850, - "fn": 550, - "accuracy": 0.9140625 - } - }, - "auroc": 0.9630278645833334 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1564, - "fn": 36, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 1501, - "fn": 99, - "accuracy": 0.938125 - } - }, - "auroc": 0.9788687174479167 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 1577, - "fn": 23, - "accuracy": 0.985625 - } - }, - "auroc": 0.9876343912760417 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3155, - "fn": 45, - "accuracy": 0.9859375 - }, - "0.01": { - "tp": 3078, - "fn": 122, - "accuracy": 0.961875 - } - }, - "auroc": 0.9832515543619792 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1421, - "fn": 179, - "accuracy": 0.888125 - }, - "0.01": { - "tp": 1223, - "fn": 377, - "accuracy": 0.764375 - } - }, - "auroc": 0.9158021809895833 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1548, - "fn": 52, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 1452, - "fn": 148, - "accuracy": 0.9075 - } - }, - "auroc": 0.9546210774739583 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2969, - "fn": 231, - "accuracy": 0.9278125 - }, - "0.01": { - "tp": 2675, - "fn": 525, - "accuracy": 0.8359375 - } - }, - "auroc": 0.9352116292317708 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2985, - "fn": 215, - "accuracy": 0.9328125 - }, - "0.01": { - "tp": 2724, - "fn": 476, - "accuracy": 0.85125 - } - }, - "auroc": 0.94733544921875 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3139, - "fn": 61, - "accuracy": 0.9809375 - }, - "0.01": { - "tp": 3029, - "fn": 171, - "accuracy": 0.9465625 - } - }, - "auroc": 0.971127734375 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 6124, - "fn": 276, - "accuracy": 0.956875 - }, - "0.01": { - "tp": 5753, - "fn": 647, - "accuracy": 0.89890625 - } - }, - "auroc": 0.959231591796875 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9948111979166667 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1598, - "fn": 2, - "accuracy": 0.99875 - }, - "0.01": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - } - }, - "auroc": 0.9941742838541666 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3194, - "fn": 6, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 3189, - "fn": 11, - "accuracy": 0.9965625 - } - }, - "auroc": 0.9944927408854166 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - } - }, - "auroc": 0.99145107421875 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1567, - "fn": 33, - "accuracy": 0.979375 - }, - "0.01": { - "tp": 1533, - "fn": 67, - "accuracy": 0.958125 - } - }, - "auroc": 0.9775308268229167 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3156, - "fn": 44, - "accuracy": 0.98625 - }, - "0.01": { - "tp": 3117, - "fn": 83, - "accuracy": 0.9740625 - } - }, - "auroc": 0.9844909505208334 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3185, - "fn": 15, - "accuracy": 0.9953125 - }, - "0.01": { - "tp": 3179, - "fn": 21, - "accuracy": 0.9934375 - } - }, - "auroc": 0.9931311360677084 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3165, - "fn": 35, - "accuracy": 0.9890625 - }, - "0.01": { - "tp": 3127, - "fn": 73, - "accuracy": 0.9771875 - } - }, - "auroc": 0.9858525553385415 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 6350, - "fn": 50, - "accuracy": 0.9921875 - }, - "0.01": { - "tp": 6306, - "fn": 94, - "accuracy": 0.9853125 - } - }, - "auroc": 0.9894918457031251 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1553, - "fn": 47, - "accuracy": 0.970625 - }, - "0.01": { - "tp": 1496, - "fn": 104, - "accuracy": 0.935 - } - }, - "auroc": 0.9702215169270834 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1553, - "fn": 47, - "accuracy": 0.970625 - }, - "0.01": { - "tp": 1496, - "fn": 104, - "accuracy": 0.935 - } - }, - "auroc": 0.9702215169270834 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1536, - "fn": 64, - "accuracy": 0.96 - }, - "0.01": { - "tp": 1458, - "fn": 142, - "accuracy": 0.91125 - } - }, - "auroc": 0.960076806640625 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1536, - "fn": 64, - "accuracy": 0.96 - }, - "0.01": { - "tp": 1458, - "fn": 142, - "accuracy": 0.91125 - } - }, - "auroc": 0.960076806640625 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3089, - "fn": 111, - "accuracy": 0.9653125 - }, - "0.01": { - "tp": 2954, - "fn": 246, - "accuracy": 0.923125 - } - }, - "auroc": 0.9651491617838542 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3089, - "fn": 111, - "accuracy": 0.9653125 - }, - "0.01": { - "tp": 2954, - "fn": 246, - "accuracy": 0.923125 - } - }, - "auroc": 0.9651491617838542 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1502, - "fn": 98, - "accuracy": 0.93875 - }, - "0.01": { - "tp": 1388, - "fn": 212, - "accuracy": 0.8675 - } - }, - "auroc": 0.9432201171875001 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1502, - "fn": 98, - "accuracy": 0.93875 - }, - "0.01": { - "tp": 1388, - "fn": 212, - "accuracy": 0.8675 - } - }, - "auroc": 0.9432201171875001 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1443, - "fn": 157, - "accuracy": 0.901875 - }, - "0.01": { - "tp": 1285, - "fn": 315, - "accuracy": 0.803125 - } - }, - "auroc": 0.9240839029947916 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1443, - "fn": 157, - "accuracy": 0.901875 - }, - "0.01": { - "tp": 1285, - "fn": 315, - "accuracy": 0.803125 - } - }, - "auroc": 0.9240839029947916 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2945, - "fn": 255, - "accuracy": 0.9203125 - }, - "0.01": { - "tp": 2673, - "fn": 527, - "accuracy": 0.8353125 - } - }, - "auroc": 0.9336520100911458 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 2945, - "fn": 255, - "accuracy": 0.9203125 - }, - "0.01": { - "tp": 2673, - "fn": 527, - "accuracy": 0.8353125 - } - }, - "auroc": 0.9336520100911458 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.992280712890625 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.992280712890625 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - } - }, - "auroc": 0.990135791015625 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - } - }, - "auroc": 0.990135791015625 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3186, - "fn": 14, - "accuracy": 0.995625 - } - }, - "auroc": 0.9912082519531249 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3186, - "fn": 14, - "accuracy": 0.995625 - } - }, - "auroc": 0.9912082519531249 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - } - }, - "auroc": 0.9937243815104166 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - }, - "0.01": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - } - }, - "auroc": 0.9937243815104166 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1593, - "fn": 7, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 1567, - "fn": 33, - "accuracy": 0.979375 - } - }, - "auroc": 0.9795080078125 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1593, - "fn": 7, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 1567, - "fn": 33, - "accuracy": 0.979375 - } - }, - "auroc": 0.9795080078125 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3192, - "fn": 8, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 3164, - "fn": 36, - "accuracy": 0.98875 - } - }, - "auroc": 0.9866161946614584 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3192, - "fn": 8, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 3164, - "fn": 36, - "accuracy": 0.98875 - } - }, - "auroc": 0.9866161946614584 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1532, - "fn": 68, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 1472, - "fn": 128, - "accuracy": 0.92 - } - }, - "auroc": 0.9686684733072917 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1532, - "fn": 68, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 1472, - "fn": 128, - "accuracy": 0.92 - } - }, - "auroc": 0.9686684733072917 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1486, - "fn": 114, - "accuracy": 0.92875 - }, - "0.01": { - "tp": 1390, - "fn": 210, - "accuracy": 0.86875 - } - }, - "auroc": 0.9520654785156251 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 1486, - "fn": 114, - "accuracy": 0.92875 - }, - "0.01": { - "tp": 1390, - "fn": 210, - "accuracy": 0.86875 - } - }, - "auroc": 0.9520654785156251 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3018, - "fn": 182, - "accuracy": 0.943125 - }, - "0.01": { - "tp": 2862, - "fn": 338, - "accuracy": 0.894375 - } - }, - "auroc": 0.9603669759114584 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 3018, - "fn": 182, - "accuracy": 0.943125 - }, - "0.01": { - "tp": 2862, - "fn": 338, - "accuracy": 0.894375 - } - }, - "auroc": 0.9603669759114584 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 17213, - "fn": 387, - "accuracy": 0.9780113636363637 - }, - "0.01": { - "tp": 16725, - "fn": 875, - "accuracy": 0.9502840909090909 - } - }, - "auroc": 0.9778130918560606 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 9549, - "fn": 51, - "accuracy": 0.9946875 - }, - "0.01": { - "tp": 9481, - "fn": 119, - "accuracy": 0.9876041666666666 - } - }, - "auroc": 0.9906324978298611 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 26762, - "fn": 438, - "accuracy": 0.9838970588235294 - }, - "0.01": { - "tp": 26206, - "fn": 994, - "accuracy": 0.9634558823529412 - } - }, - "auroc": 0.9823375880821078 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 16876, - "fn": 724, - "accuracy": 0.9588636363636364 - }, - "0.01": { - "tp": 16009, - "fn": 1591, - "accuracy": 0.9096022727272727 - } - }, - "auroc": 0.9624637650923294 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 9386, - "fn": 214, - "accuracy": 0.9777083333333333 - }, - "0.01": { - "tp": 9082, - "fn": 518, - "accuracy": 0.9460416666666667 - } - }, - "auroc": 0.9703925672743056 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 26262, - "fn": 938, - "accuracy": 0.9655147058823529 - }, - "0.01": { - "tp": 25091, - "fn": 2109, - "accuracy": 0.9224632352941177 - } - }, - "auroc": 0.9652621658624387 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 34089, - "fn": 1111, - "accuracy": 0.9684375 - }, - "0.01": { - "tp": 32734, - "fn": 2466, - "accuracy": 0.9299431818181818 - } - }, - "auroc": 0.9701384284741951 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 18935, - "fn": 265, - "accuracy": 0.9861979166666667 - }, - "0.01": { - "tp": 18563, - "fn": 637, - "accuracy": 0.9668229166666666 - } - }, - "auroc": 0.9805125325520834 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "alternative_spelling", - "accuracy": { - "0.05": { - "tp": 53024, - "fn": 1376, - "accuracy": 0.9747058823529412 - }, - "0.01": { - "tp": 51297, - "fn": 3103, - "accuracy": 0.9429595588235294 - } - }, - "auroc": 0.9737998769722732 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - } - }, - "auroc": 0.9965804524739583 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1599, - "fn": 1, - "accuracy": 0.999375 - } - }, - "auroc": 0.995021923828125 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3198, - "fn": 2, - "accuracy": 0.999375 - } - }, - "auroc": 0.9958011881510417 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1600, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - } - }, - "auroc": 0.995738916015625 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1580, - "fn": 20, - "accuracy": 0.9875 - } - }, - "auroc": 0.9898985351562499 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3197, - "fn": 3, - "accuracy": 0.9990625 - }, - "0.01": { - "tp": 3177, - "fn": 23, - "accuracy": 0.9928125 - } - }, - "auroc": 0.9928187255859375 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3200, - "fn": 0, - "accuracy": 1.0 - }, - "0.01": { - "tp": 3196, - "fn": 4, - "accuracy": 0.99875 - } - }, - "auroc": 0.9961596842447917 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3197, - "fn": 3, - "accuracy": 0.9990625 - }, - "0.01": { - "tp": 3179, - "fn": 21, - "accuracy": 0.9934375 - } - }, - "auroc": 0.9924602294921875 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 6397, - "fn": 3, - "accuracy": 0.99953125 - }, - "0.01": { - "tp": 6375, - "fn": 25, - "accuracy": 0.99609375 - } - }, - "auroc": 0.9943099568684897 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1564, - "fn": 36, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 1491, - "fn": 109, - "accuracy": 0.931875 - } - }, - "auroc": 0.972055859375 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - }, - "0.01": { - "tp": 1554, - "fn": 46, - "accuracy": 0.97125 - } - }, - "auroc": 0.9870034830729166 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3140, - "fn": 60, - "accuracy": 0.98125 - }, - "0.01": { - "tp": 3045, - "fn": 155, - "accuracy": 0.9515625 - } - }, - "auroc": 0.9795296712239583 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1462, - "fn": 138, - "accuracy": 0.91375 - }, - "0.01": { - "tp": 1299, - "fn": 301, - "accuracy": 0.811875 - } - }, - "auroc": 0.9275415201822916 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1534, - "fn": 66, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 1483, - "fn": 117, - "accuracy": 0.926875 - } - }, - "auroc": 0.9657855468750001 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2996, - "fn": 204, - "accuracy": 0.93625 - }, - "0.01": { - "tp": 2782, - "fn": 418, - "accuracy": 0.869375 - } - }, - "auroc": 0.9466635335286457 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3026, - "fn": 174, - "accuracy": 0.945625 - }, - "0.01": { - "tp": 2790, - "fn": 410, - "accuracy": 0.871875 - } - }, - "auroc": 0.9497986897786458 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3110, - "fn": 90, - "accuracy": 0.971875 - }, - "0.01": { - "tp": 3037, - "fn": 163, - "accuracy": 0.9490625 - } - }, - "auroc": 0.9763945149739582 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 6136, - "fn": 264, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 5827, - "fn": 573, - "accuracy": 0.91046875 - } - }, - "auroc": 0.9630966023763021 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - }, - "0.01": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - } - }, - "auroc": 0.9928874837239583 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1590, - "fn": 10, - "accuracy": 0.99375 - }, - "0.01": { - "tp": 1577, - "fn": 23, - "accuracy": 0.985625 - } - }, - "auroc": 0.9868442545572917 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3181, - "fn": 19, - "accuracy": 0.9940625 - }, - "0.01": { - "tp": 3161, - "fn": 39, - "accuracy": 0.9878125 - } - }, - "auroc": 0.989865869140625 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1588, - "fn": 12, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 1576, - "fn": 24, - "accuracy": 0.985 - } - }, - "auroc": 0.990519140625 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1584, - "fn": 16, - "accuracy": 0.99 - }, - "0.01": { - "tp": 1524, - "fn": 76, - "accuracy": 0.9525 - } - }, - "auroc": 0.9697653483072917 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3172, - "fn": 28, - "accuracy": 0.99125 - }, - "0.01": { - "tp": 3100, - "fn": 100, - "accuracy": 0.96875 - } - }, - "auroc": 0.9801422444661458 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3179, - "fn": 21, - "accuracy": 0.9934375 - }, - "0.01": { - "tp": 3160, - "fn": 40, - "accuracy": 0.9875 - } - }, - "auroc": 0.9917033121744792 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3174, - "fn": 26, - "accuracy": 0.991875 - }, - "0.01": { - "tp": 3101, - "fn": 99, - "accuracy": 0.9690625 - } - }, - "auroc": 0.9783048014322916 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 6353, - "fn": 47, - "accuracy": 0.99265625 - }, - "0.01": { - "tp": 6261, - "fn": 139, - "accuracy": 0.97828125 - } - }, - "auroc": 0.9850040568033854 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1518, - "fn": 82, - "accuracy": 0.94875 - }, - "0.01": { - "tp": 1403, - "fn": 197, - "accuracy": 0.876875 - } - }, - "auroc": 0.952021923828125 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 1571, - "fn": 29, - "accuracy": 0.981875 - } - }, - "auroc": 0.991476220703125 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3113, - "fn": 87, - "accuracy": 0.9728125 - }, - "0.01": { - "tp": 2974, - "fn": 226, - "accuracy": 0.929375 - } - }, - "auroc": 0.971749072265625 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1512, - "fn": 88, - "accuracy": 0.945 - }, - "0.01": { - "tp": 1285, - "fn": 315, - "accuracy": 0.803125 - } - }, - "auroc": 0.9330303873697916 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1551, - "fn": 49, - "accuracy": 0.969375 - }, - "0.01": { - "tp": 1488, - "fn": 112, - "accuracy": 0.93 - } - }, - "auroc": 0.9527174479166667 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3063, - "fn": 137, - "accuracy": 0.9571875 - }, - "0.01": { - "tp": 2773, - "fn": 427, - "accuracy": 0.8665625 - } - }, - "auroc": 0.9428739176432291 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3030, - "fn": 170, - "accuracy": 0.946875 - }, - "0.01": { - "tp": 2688, - "fn": 512, - "accuracy": 0.84 - } - }, - "auroc": 0.9425261555989584 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3146, - "fn": 54, - "accuracy": 0.983125 - }, - "0.01": { - "tp": 3059, - "fn": 141, - "accuracy": 0.9559375 - } - }, - "auroc": 0.9720968343098957 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 6176, - "fn": 224, - "accuracy": 0.965 - }, - "0.01": { - "tp": 5747, - "fn": 653, - "accuracy": 0.89796875 - } - }, - "auroc": 0.957311494954427 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1565, - "fn": 35, - "accuracy": 0.978125 - }, - "0.01": { - "tp": 1502, - "fn": 98, - "accuracy": 0.93875 - } - }, - "auroc": 0.9784849772135416 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1572, - "fn": 28, - "accuracy": 0.9825 - } - }, - "auroc": 0.9847144856770833 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3154, - "fn": 46, - "accuracy": 0.985625 - }, - "0.01": { - "tp": 3074, - "fn": 126, - "accuracy": 0.960625 - } - }, - "auroc": 0.9815997314453125 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1361, - "fn": 239, - "accuracy": 0.850625 - }, - "0.01": { - "tp": 1129, - "fn": 471, - "accuracy": 0.705625 - } - }, - "auroc": 0.897958544921875 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1534, - "fn": 66, - "accuracy": 0.95875 - }, - "0.01": { - "tp": 1433, - "fn": 167, - "accuracy": 0.895625 - } - }, - "auroc": 0.949140869140625 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2895, - "fn": 305, - "accuracy": 0.9046875 - }, - "0.01": { - "tp": 2562, - "fn": 638, - "accuracy": 0.800625 - } - }, - "auroc": 0.92354970703125 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2926, - "fn": 274, - "accuracy": 0.914375 - }, - "0.01": { - "tp": 2631, - "fn": 569, - "accuracy": 0.8221875 - } - }, - "auroc": 0.9382217610677083 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3123, - "fn": 77, - "accuracy": 0.9759375 - }, - "0.01": { - "tp": 3005, - "fn": 195, - "accuracy": 0.9390625 - } - }, - "auroc": 0.9669276774088542 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 6049, - "fn": 351, - "accuracy": 0.94515625 - }, - "0.01": { - "tp": 5636, - "fn": 764, - "accuracy": 0.880625 - } - }, - "auroc": 0.9525747192382813 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.99486494140625 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1593, - "fn": 7, - "accuracy": 0.995625 - } - }, - "auroc": 0.9939198567708333 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3193, - "fn": 7, - "accuracy": 0.9978125 - }, - "0.01": { - "tp": 3188, - "fn": 12, - "accuracy": 0.99625 - } - }, - "auroc": 0.9943923990885417 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1588, - "fn": 12, - "accuracy": 0.9925 - }, - "0.01": { - "tp": 1582, - "fn": 18, - "accuracy": 0.98875 - } - }, - "auroc": 0.9908359863281251 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1564, - "fn": 36, - "accuracy": 0.9775 - }, - "0.01": { - "tp": 1522, - "fn": 78, - "accuracy": 0.95125 - } - }, - "auroc": 0.974113525390625 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3152, - "fn": 48, - "accuracy": 0.985 - }, - "0.01": { - "tp": 3104, - "fn": 96, - "accuracy": 0.97 - } - }, - "auroc": 0.9824747558593749 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3184, - "fn": 16, - "accuracy": 0.995 - }, - "0.01": { - "tp": 3177, - "fn": 23, - "accuracy": 0.9928125 - } - }, - "auroc": 0.9928504638671874 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3161, - "fn": 39, - "accuracy": 0.9878125 - }, - "0.01": { - "tp": 3115, - "fn": 85, - "accuracy": 0.9734375 - } - }, - "auroc": 0.9840166910807292 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 6345, - "fn": 55, - "accuracy": 0.99140625 - }, - "0.01": { - "tp": 6292, - "fn": 108, - "accuracy": 0.983125 - } - }, - "auroc": 0.9884335774739583 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1548, - "fn": 52, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 1490, - "fn": 110, - "accuracy": 0.93125 - } - }, - "auroc": 0.9677334798177082 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1548, - "fn": 52, - "accuracy": 0.9675 - }, - "0.01": { - "tp": 1490, - "fn": 110, - "accuracy": 0.93125 - } - }, - "auroc": 0.9677334798177082 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1527, - "fn": 73, - "accuracy": 0.954375 - }, - "0.01": { - "tp": 1450, - "fn": 150, - "accuracy": 0.90625 - } - }, - "auroc": 0.9576980143229167 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1527, - "fn": 73, - "accuracy": 0.954375 - }, - "0.01": { - "tp": 1450, - "fn": 150, - "accuracy": 0.90625 - } - }, - "auroc": 0.9576980143229167 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3075, - "fn": 125, - "accuracy": 0.9609375 - }, - "0.01": { - "tp": 2940, - "fn": 260, - "accuracy": 0.91875 - } - }, - "auroc": 0.9627157470703125 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3075, - "fn": 125, - "accuracy": 0.9609375 - }, - "0.01": { - "tp": 2940, - "fn": 260, - "accuracy": 0.91875 - } - }, - "auroc": 0.9627157470703125 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1484, - "fn": 116, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 1346, - "fn": 254, - "accuracy": 0.84125 - } - }, - "auroc": 0.9343791015625 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1484, - "fn": 116, - "accuracy": 0.9275 - }, - "0.01": { - "tp": 1346, - "fn": 254, - "accuracy": 0.84125 - } - }, - "auroc": 0.9343791015625 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1401, - "fn": 199, - "accuracy": 0.875625 - }, - "0.01": { - "tp": 1225, - "fn": 375, - "accuracy": 0.765625 - } - }, - "auroc": 0.9109469889322916 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1401, - "fn": 199, - "accuracy": 0.875625 - }, - "0.01": { - "tp": 1225, - "fn": 375, - "accuracy": 0.765625 - } - }, - "auroc": 0.9109469889322916 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2885, - "fn": 315, - "accuracy": 0.9015625 - }, - "0.01": { - "tp": 2571, - "fn": 629, - "accuracy": 0.8034375 - } - }, - "auroc": 0.9226630452473958 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2885, - "fn": 315, - "accuracy": 0.9015625 - }, - "0.01": { - "tp": 2571, - "fn": 629, - "accuracy": 0.8034375 - } - }, - "auroc": 0.9226630452473958 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9924631184895834 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 1596, - "fn": 4, - "accuracy": 0.9975 - } - }, - "auroc": 0.9924631184895834 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - } - }, - "auroc": 0.990256982421875 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1594, - "fn": 6, - "accuracy": 0.99625 - }, - "0.01": { - "tp": 1591, - "fn": 9, - "accuracy": 0.994375 - } - }, - "auroc": 0.990256982421875 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3187, - "fn": 13, - "accuracy": 0.9959375 - } - }, - "auroc": 0.9913600504557292 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3190, - "fn": 10, - "accuracy": 0.996875 - }, - "0.01": { - "tp": 3187, - "fn": 13, - "accuracy": 0.9959375 - } - }, - "auroc": 0.9913600504557292 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9935093912760417 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1597, - "fn": 3, - "accuracy": 0.998125 - }, - "0.01": { - "tp": 1595, - "fn": 5, - "accuracy": 0.996875 - } - }, - "auroc": 0.9935093912760417 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1552, - "fn": 48, - "accuracy": 0.97 - } - }, - "auroc": 0.9767893391927083 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1589, - "fn": 11, - "accuracy": 0.993125 - }, - "0.01": { - "tp": 1552, - "fn": 48, - "accuracy": 0.97 - } - }, - "auroc": 0.9767893391927083 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3186, - "fn": 14, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 3147, - "fn": 53, - "accuracy": 0.9834375 - } - }, - "auroc": 0.985149365234375 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 3186, - "fn": 14, - "accuracy": 0.995625 - }, - "0.01": { - "tp": 3147, - "fn": 53, - "accuracy": 0.9834375 - } - }, - "auroc": 0.985149365234375 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1521, - "fn": 79, - "accuracy": 0.950625 - }, - "0.01": { - "tp": 1448, - "fn": 152, - "accuracy": 0.905 - } - }, - "auroc": 0.9640538248697916 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1521, - "fn": 79, - "accuracy": 0.950625 - }, - "0.01": { - "tp": 1448, - "fn": 152, - "accuracy": 0.905 - } - }, - "auroc": 0.9640538248697916 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1468, - "fn": 132, - "accuracy": 0.9175 - }, - "0.01": { - "tp": 1371, - "fn": 229, - "accuracy": 0.856875 - } - }, - "auroc": 0.9452570963541667 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 1468, - "fn": 132, - "accuracy": 0.9175 - }, - "0.01": { - "tp": 1371, - "fn": 229, - "accuracy": 0.856875 - } - }, - "auroc": 0.9452570963541667 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2989, - "fn": 211, - "accuracy": 0.9340625 - }, - "0.01": { - "tp": 2819, - "fn": 381, - "accuracy": 0.8809375 - } - }, - "auroc": 0.9546554606119791 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 2989, - "fn": 211, - "accuracy": 0.9340625 - }, - "0.01": { - "tp": 2819, - "fn": 381, - "accuracy": 0.8809375 - } - }, - "auroc": 0.9546554606119791 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 17180, - "fn": 420, - "accuracy": 0.9761363636363637 - }, - "0.01": { - "tp": 16649, - "fn": 951, - "accuracy": 0.945965909090909 - } - }, - "auroc": 0.976275868548769 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 9547, - "fn": 53, - "accuracy": 0.9944791666666667 - }, - "0.01": { - "tp": 9466, - "fn": 134, - "accuracy": 0.9860416666666667 - } - }, - "auroc": 0.9898300374348958 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 26727, - "fn": 473, - "accuracy": 0.9826102941176471 - }, - "0.01": { - "tp": 26115, - "fn": 1085, - "accuracy": 0.960110294117647 - } - }, - "auroc": 0.9810596928615196 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 16690, - "fn": 910, - "accuracy": 0.9482954545454545 - }, - "0.01": { - "tp": 15657, - "fn": 1943, - "accuracy": 0.8896022727272728 - } - }, - "auroc": 0.9560520833333334 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 9364, - "fn": 236, - "accuracy": 0.9754166666666667 - }, - "0.01": { - "tp": 9030, - "fn": 570, - "accuracy": 0.940625 - } - }, - "auroc": 0.9669035454644097 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 26054, - "fn": 1146, - "accuracy": 0.9578676470588235 - }, - "0.01": { - "tp": 24687, - "fn": 2513, - "accuracy": 0.907610294117647 - } - }, - "auroc": 0.9598820111443014 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 33870, - "fn": 1330, - "accuracy": 0.962215909090909 - }, - "0.01": { - "tp": 32306, - "fn": 2894, - "accuracy": 0.9177840909090909 - } - }, - "auroc": 0.9661639759410511 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 18911, - "fn": 289, - "accuracy": 0.9849479166666667 - }, - "0.01": { - "tp": 18496, - "fn": 704, - "accuracy": 0.9633333333333334 - } - }, - "auroc": 0.9783667914496528 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "zero_width_space", - "accuracy": { - "0.05": { - "tp": 52781, - "fn": 1619, - "accuracy": 0.9702389705882353 - }, - "0.01": { - "tp": 50802, - "fn": 3598, - "accuracy": 0.933860294117647 - } - }, - "auroc": 0.9704708520029104 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19199, - "fn": 1, - "accuracy": 0.9999479166666667 - }, - "0.01": { - "tp": 19165, - "fn": 35, - "accuracy": 0.9981770833333333 - } - }, - "auroc": 0.9917713745117187 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19194, - "fn": 6, - "accuracy": 0.9996875 - }, - "0.01": { - "tp": 19126, - "fn": 74, - "accuracy": 0.9961458333333333 - } - }, - "auroc": 0.9889261773003473 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 38393, - "fn": 7, - "accuracy": 0.9998177083333334 - }, - "0.01": { - "tp": 38291, - "fn": 109, - "accuracy": 0.9971614583333334 - } - }, - "auroc": 0.990348775906033 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19198, - "fn": 2, - "accuracy": 0.9998958333333333 - }, - "0.01": { - "tp": 19136, - "fn": 64, - "accuracy": 0.9966666666666667 - } - }, - "auroc": 0.9910077094184029 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19160, - "fn": 40, - "accuracy": 0.9979166666666667 - }, - "0.01": { - "tp": 18923, - "fn": 277, - "accuracy": 0.9855729166666667 - } - }, - "auroc": 0.9841562798394097 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 38358, - "fn": 42, - "accuracy": 0.99890625 - }, - "0.01": { - "tp": 38059, - "fn": 341, - "accuracy": 0.9911197916666666 - } - }, - "auroc": 0.9875819946289062 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 38397, - "fn": 3, - "accuracy": 0.999921875 - }, - "0.01": { - "tp": 38301, - "fn": 99, - "accuracy": 0.997421875 - } - }, - "auroc": 0.9913895419650608 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 38354, - "fn": 46, - "accuracy": 0.9988020833333333 - }, - "0.01": { - "tp": 38049, - "fn": 351, - "accuracy": 0.990859375 - } - }, - "auroc": 0.9865412285698785 - }, - { - "domain": "all", - "model": "llama-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 76751, - "fn": 49, - "accuracy": 0.9993619791666667 - }, - "0.01": { - "tp": 76350, - "fn": 450, - "accuracy": 0.994140625 - } - }, - "auroc": 0.9889653852674697 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18765, - "fn": 435, - "accuracy": 0.97734375 - }, - "0.01": { - "tp": 17871, - "fn": 1329, - "accuracy": 0.93078125 - } - }, - "auroc": 0.9714085259331597 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18743, - "fn": 457, - "accuracy": 0.9761979166666667 - }, - "0.01": { - "tp": 18177, - "fn": 1023, - "accuracy": 0.94671875 - } - }, - "auroc": 0.9775881401909723 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 37508, - "fn": 892, - "accuracy": 0.9767708333333334 - }, - "0.01": { - "tp": 36048, - "fn": 2352, - "accuracy": 0.93875 - } - }, - "auroc": 0.974498333062066 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18069, - "fn": 1131, - "accuracy": 0.94109375 - }, - "0.01": { - "tp": 16589, - "fn": 2611, - "accuracy": 0.8640104166666667 - } - }, - "auroc": 0.9411877495659722 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18079, - "fn": 1121, - "accuracy": 0.9416145833333334 - }, - "0.01": { - "tp": 17147, - "fn": 2053, - "accuracy": 0.8930729166666667 - } - }, - "auroc": 0.9554624267578126 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 36148, - "fn": 2252, - "accuracy": 0.9413541666666667 - }, - "0.01": { - "tp": 33736, - "fn": 4664, - "accuracy": 0.8785416666666667 - } - }, - "auroc": 0.9483250881618924 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 36834, - "fn": 1566, - "accuracy": 0.95921875 - }, - "0.01": { - "tp": 34460, - "fn": 3940, - "accuracy": 0.8973958333333333 - } - }, - "auroc": 0.9562981377495658 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 36822, - "fn": 1578, - "accuracy": 0.95890625 - }, - "0.01": { - "tp": 35324, - "fn": 3076, - "accuracy": 0.9198958333333334 - } - }, - "auroc": 0.9665252834743924 - }, - { - "domain": "all", - "model": "mpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 73656, - "fn": 3144, - "accuracy": 0.9590625 - }, - "0.01": { - "tp": 69784, - "fn": 7016, - "accuracy": 0.9086458333333334 - } - }, - "auroc": 0.9614117106119792 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19079, - "fn": 121, - "accuracy": 0.9936979166666666 - }, - "0.01": { - "tp": 18916, - "fn": 284, - "accuracy": 0.9852083333333334 - } - }, - "auroc": 0.9878157375759549 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19023, - "fn": 177, - "accuracy": 0.99078125 - }, - "0.01": { - "tp": 18700, - "fn": 500, - "accuracy": 0.9739583333333334 - } - }, - "auroc": 0.9796440687391494 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 38102, - "fn": 298, - "accuracy": 0.9922395833333333 - }, - "0.01": { - "tp": 37616, - "fn": 784, - "accuracy": 0.9795833333333334 - } - }, - "auroc": 0.9837299031575522 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19027, - "fn": 173, - "accuracy": 0.9909895833333333 - }, - "0.01": { - "tp": 18799, - "fn": 401, - "accuracy": 0.9791145833333333 - } - }, - "auroc": 0.9849998806423611 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18886, - "fn": 314, - "accuracy": 0.9836458333333333 - }, - "0.01": { - "tp": 18010, - "fn": 1190, - "accuracy": 0.9380208333333333 - } - }, - "auroc": 0.9642955824110242 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 37913, - "fn": 487, - "accuracy": 0.9873177083333333 - }, - "0.01": { - "tp": 36809, - "fn": 1591, - "accuracy": 0.9585677083333334 - } - }, - "auroc": 0.9746477315266927 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 38106, - "fn": 294, - "accuracy": 0.99234375 - }, - "0.01": { - "tp": 37715, - "fn": 685, - "accuracy": 0.9821614583333333 - } - }, - "auroc": 0.9864078091091579 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 37909, - "fn": 491, - "accuracy": 0.9872135416666666 - }, - "0.01": { - "tp": 36710, - "fn": 1690, - "accuracy": 0.9559895833333333 - } - }, - "auroc": 0.9719698255750868 - }, - { - "domain": "all", - "model": "mpt-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 76015, - "fn": 785, - "accuracy": 0.9897786458333333 - }, - "0.01": { - "tp": 74425, - "fn": 2375, - "accuracy": 0.9690755208333334 - } - }, - "auroc": 0.9791888173421224 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18196, - "fn": 1004, - "accuracy": 0.9477083333333334 - }, - "0.01": { - "tp": 16696, - "fn": 2504, - "accuracy": 0.8695833333333334 - } - }, - "auroc": 0.9508460300021702 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19121, - "fn": 79, - "accuracy": 0.9958854166666666 - }, - "0.01": { - "tp": 18833, - "fn": 367, - "accuracy": 0.9808854166666666 - } - }, - "auroc": 0.9885517835828993 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 37317, - "fn": 1083, - "accuracy": 0.971796875 - }, - "0.01": { - "tp": 35529, - "fn": 2871, - "accuracy": 0.925234375 - } - }, - "auroc": 0.9696989067925347 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18412, - "fn": 788, - "accuracy": 0.9589583333333334 - }, - "0.01": { - "tp": 16406, - "fn": 2794, - "accuracy": 0.8544791666666667 - } - }, - "auroc": 0.9437618082682292 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18579, - "fn": 621, - "accuracy": 0.96765625 - }, - "0.01": { - "tp": 17654, - "fn": 1546, - "accuracy": 0.9194791666666666 - } - }, - "auroc": 0.9550365709092882 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 36991, - "fn": 1409, - "accuracy": 0.9633072916666666 - }, - "0.01": { - "tp": 34060, - "fn": 4340, - "accuracy": 0.8869791666666667 - } - }, - "auroc": 0.9493991895887586 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 36608, - "fn": 1792, - "accuracy": 0.9533333333333334 - }, - "0.01": { - "tp": 33102, - "fn": 5298, - "accuracy": 0.86203125 - } - }, - "auroc": 0.9473039191351997 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 37700, - "fn": 700, - "accuracy": 0.9817708333333334 - }, - "0.01": { - "tp": 36487, - "fn": 1913, - "accuracy": 0.9501822916666667 - } - }, - "auroc": 0.9717941772460937 - }, - { - "domain": "all", - "model": "gpt2", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 74308, - "fn": 2492, - "accuracy": 0.9675520833333333 - }, - "0.01": { - "tp": 69589, - "fn": 7211, - "accuracy": 0.9061067708333334 - } - }, - "auroc": 0.9595490481906468 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18772, - "fn": 428, - "accuracy": 0.9777083333333333 - }, - "0.01": { - "tp": 17987, - "fn": 1213, - "accuracy": 0.9368229166666666 - } - }, - "auroc": 0.9769103488498263 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19008, - "fn": 192, - "accuracy": 0.99 - }, - "0.01": { - "tp": 18652, - "fn": 548, - "accuracy": 0.9714583333333333 - } - }, - "auroc": 0.9801447658962674 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 37780, - "fn": 620, - "accuracy": 0.9838541666666667 - }, - "0.01": { - "tp": 36639, - "fn": 1761, - "accuracy": 0.954140625 - } - }, - "auroc": 0.9785275573730469 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 17298, - "fn": 1902, - "accuracy": 0.9009375 - }, - "0.01": { - "tp": 14887, - "fn": 4313, - "accuracy": 0.7753645833333334 - } - }, - "auroc": 0.9168898857964409 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18264, - "fn": 936, - "accuracy": 0.95125 - }, - "0.01": { - "tp": 16911, - "fn": 2289, - "accuracy": 0.88078125 - } - }, - "auroc": 0.9458360677083333 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 35562, - "fn": 2838, - "accuracy": 0.92609375 - }, - "0.01": { - "tp": 31798, - "fn": 6602, - "accuracy": 0.8280729166666667 - } - }, - "auroc": 0.9313629767523872 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 36070, - "fn": 2330, - "accuracy": 0.9393229166666667 - }, - "0.01": { - "tp": 32874, - "fn": 5526, - "accuracy": 0.85609375 - } - }, - "auroc": 0.9469001173231336 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 37272, - "fn": 1128, - "accuracy": 0.970625 - }, - "0.01": { - "tp": 35563, - "fn": 2837, - "accuracy": 0.9261197916666667 - } - }, - "auroc": 0.9629904168023002 - }, - { - "domain": "all", - "model": "mistral", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 73342, - "fn": 3458, - "accuracy": 0.9549739583333333 - }, - "0.01": { - "tp": 68437, - "fn": 8363, - "accuracy": 0.8911067708333333 - } - }, - "auroc": 0.9549452670627171 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19144, - "fn": 56, - "accuracy": 0.9970833333333333 - }, - "0.01": { - "tp": 19085, - "fn": 115, - "accuracy": 0.9940104166666667 - } - }, - "auroc": 0.9909244588216146 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19152, - "fn": 48, - "accuracy": 0.9975 - }, - "0.01": { - "tp": 19049, - "fn": 151, - "accuracy": 0.9921354166666667 - } - }, - "auroc": 0.988433467610677 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 38296, - "fn": 104, - "accuracy": 0.9972916666666667 - }, - "0.01": { - "tp": 38134, - "fn": 266, - "accuracy": 0.9930729166666666 - } - }, - "auroc": 0.9896789632161459 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19049, - "fn": 151, - "accuracy": 0.9921354166666667 - }, - "0.01": { - "tp": 18917, - "fn": 283, - "accuracy": 0.9852604166666666 - } - }, - "auroc": 0.9863227389865451 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18837, - "fn": 363, - "accuracy": 0.98109375 - }, - "0.01": { - "tp": 18302, - "fn": 898, - "accuracy": 0.9532291666666667 - } - }, - "auroc": 0.9720264051649306 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 37886, - "fn": 514, - "accuracy": 0.9866145833333333 - }, - "0.01": { - "tp": 37219, - "fn": 1181, - "accuracy": 0.9692447916666667 - } - }, - "auroc": 0.9791745720757379 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 38193, - "fn": 207, - "accuracy": 0.994609375 - }, - "0.01": { - "tp": 38002, - "fn": 398, - "accuracy": 0.9896354166666667 - } - }, - "auroc": 0.9886235989040799 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 37989, - "fn": 411, - "accuracy": 0.989296875 - }, - "0.01": { - "tp": 37351, - "fn": 1049, - "accuracy": 0.9726822916666666 - } - }, - "auroc": 0.9802299363878038 - }, - { - "domain": "all", - "model": "mistral-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 76182, - "fn": 618, - "accuracy": 0.991953125 - }, - "0.01": { - "tp": 75353, - "fn": 1447, - "accuracy": 0.9811588541666667 - } - }, - "auroc": 0.984426767645942 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18516, - "fn": 684, - "accuracy": 0.964375 - }, - "0.01": { - "tp": 17696, - "fn": 1504, - "accuracy": 0.9216666666666666 - } - }, - "auroc": 0.9631309244791667 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18516, - "fn": 684, - "accuracy": 0.964375 - }, - "0.01": { - "tp": 17696, - "fn": 1504, - "accuracy": 0.9216666666666666 - } - }, - "auroc": 0.9631309244791667 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18288, - "fn": 912, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 17256, - "fn": 1944, - "accuracy": 0.89875 - } - }, - "auroc": 0.9538452785915799 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18288, - "fn": 912, - "accuracy": 0.9525 - }, - "0.01": { - "tp": 17256, - "fn": 1944, - "accuracy": 0.89875 - } - }, - "auroc": 0.9538452785915799 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 36804, - "fn": 1596, - "accuracy": 0.9584375 - }, - "0.01": { - "tp": 34952, - "fn": 3448, - "accuracy": 0.9102083333333333 - } - }, - "auroc": 0.9584881015353733 - }, - { - "domain": "all", - "model": "gpt3", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 36804, - "fn": 1596, - "accuracy": 0.9584375 - }, - "0.01": { - "tp": 34952, - "fn": 3448, - "accuracy": 0.9102083333333333 - } - }, - "auroc": 0.9584881015353733 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 17996, - "fn": 1204, - "accuracy": 0.9372916666666666 - }, - "0.01": { - "tp": 16535, - "fn": 2665, - "accuracy": 0.8611979166666667 - } - }, - "auroc": 0.9385859144422742 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 17996, - "fn": 1204, - "accuracy": 0.9372916666666666 - }, - "0.01": { - "tp": 16535, - "fn": 2665, - "accuracy": 0.8611979166666667 - } - }, - "auroc": 0.9385859144422742 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 17296, - "fn": 1904, - "accuracy": 0.9008333333333334 - }, - "0.01": { - "tp": 15335, - "fn": 3865, - "accuracy": 0.7986979166666667 - } - }, - "auroc": 0.9194236789279514 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 17296, - "fn": 1904, - "accuracy": 0.9008333333333334 - }, - "0.01": { - "tp": 15335, - "fn": 3865, - "accuracy": 0.7986979166666667 - } - }, - "auroc": 0.9194236789279514 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 35292, - "fn": 3108, - "accuracy": 0.9190625 - }, - "0.01": { - "tp": 31870, - "fn": 6530, - "accuracy": 0.8299479166666667 - } - }, - "auroc": 0.9290047966851128 - }, - { - "domain": "all", - "model": "cohere", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 35292, - "fn": 3108, - "accuracy": 0.9190625 - }, - "0.01": { - "tp": 31870, - "fn": 6530, - "accuracy": 0.8299479166666667 - } - }, - "auroc": 0.9290047966851128 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19151, - "fn": 49, - "accuracy": 0.9974479166666667 - }, - "0.01": { - "tp": 19059, - "fn": 141, - "accuracy": 0.99265625 - } - }, - "auroc": 0.9874428480360242 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19151, - "fn": 49, - "accuracy": 0.9974479166666667 - }, - "0.01": { - "tp": 19059, - "fn": 141, - "accuracy": 0.99265625 - } - }, - "auroc": 0.9874428480360242 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19125, - "fn": 75, - "accuracy": 0.99609375 - }, - "0.01": { - "tp": 19000, - "fn": 200, - "accuracy": 0.9895833333333334 - } - }, - "auroc": 0.9850551554361979 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19125, - "fn": 75, - "accuracy": 0.99609375 - }, - "0.01": { - "tp": 19000, - "fn": 200, - "accuracy": 0.9895833333333334 - } - }, - "auroc": 0.9850551554361979 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 38276, - "fn": 124, - "accuracy": 0.9967708333333334 - }, - "0.01": { - "tp": 38059, - "fn": 341, - "accuracy": 0.9911197916666666 - } - }, - "auroc": 0.9862490017361112 - }, - { - "domain": "all", - "model": "chatgpt", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 38276, - "fn": 124, - "accuracy": 0.9967708333333334 - }, - "0.01": { - "tp": 38059, - "fn": 341, - "accuracy": 0.9911197916666666 - } - }, - "auroc": 0.9862490017361112 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19166, - "fn": 34, - "accuracy": 0.9982291666666666 - }, - "0.01": { - "tp": 19078, - "fn": 122, - "accuracy": 0.9936458333333333 - } - }, - "auroc": 0.9884427625868055 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19166, - "fn": 34, - "accuracy": 0.9982291666666666 - }, - "0.01": { - "tp": 19078, - "fn": 122, - "accuracy": 0.9936458333333333 - } - }, - "auroc": 0.9884427625868055 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19085, - "fn": 115, - "accuracy": 0.9940104166666667 - }, - "0.01": { - "tp": 18618, - "fn": 582, - "accuracy": 0.9696875 - } - }, - "auroc": 0.9730088161892362 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 19085, - "fn": 115, - "accuracy": 0.9940104166666667 - }, - "0.01": { - "tp": 18618, - "fn": 582, - "accuracy": 0.9696875 - } - }, - "auroc": 0.9730088161892362 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 38251, - "fn": 149, - "accuracy": 0.9961197916666666 - }, - "0.01": { - "tp": 37696, - "fn": 704, - "accuracy": 0.9816666666666667 - } - }, - "auroc": 0.9807257893880208 - }, - { - "domain": "all", - "model": "gpt4", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 38251, - "fn": 149, - "accuracy": 0.9961197916666666 - }, - "0.01": { - "tp": 37696, - "fn": 704, - "accuracy": 0.9816666666666667 - } - }, - "auroc": 0.9807257893880208 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18384, - "fn": 816, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 17531, - "fn": 1669, - "accuracy": 0.9130729166666667 - } - }, - "auroc": 0.963472607421875 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 18384, - "fn": 816, - "accuracy": 0.9575 - }, - "0.01": { - "tp": 17531, - "fn": 1669, - "accuracy": 0.9130729166666667 - } - }, - "auroc": 0.963472607421875 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 17860, - "fn": 1340, - "accuracy": 0.9302083333333333 - }, - "0.01": { - "tp": 16628, - "fn": 2572, - "accuracy": 0.8660416666666667 - } - }, - "auroc": 0.9469042141384548 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 17860, - "fn": 1340, - "accuracy": 0.9302083333333333 - }, - "0.01": { - "tp": 16628, - "fn": 2572, - "accuracy": 0.8660416666666667 - } - }, - "auroc": 0.9469042141384548 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 36244, - "fn": 2156, - "accuracy": 0.9438541666666667 - }, - "0.01": { - "tp": 34159, - "fn": 4241, - "accuracy": 0.8895572916666666 - } - }, - "auroc": 0.9551884107801649 - }, - { - "domain": "all", - "model": "cohere-chat", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 36244, - "fn": 2156, - "accuracy": 0.9438541666666667 - }, - "0.01": { - "tp": 34159, - "fn": 4241, - "accuracy": 0.8895572916666666 - } - }, - "auroc": 0.9551884107801649 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 206368, - "fn": 4832, - "accuracy": 0.9771212121212122 - }, - "0.01": { - "tp": 199619, - "fn": 11581, - "accuracy": 0.9451657196969697 - } - }, - "auroc": 0.9737046847873264 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 114241, - "fn": 959, - "accuracy": 0.9916753472222222 - }, - "0.01": { - "tp": 112537, - "fn": 2663, - "accuracy": 0.9768836805555555 - } - }, - "auroc": 0.9838814005533854 - }, - { - "domain": "all", - "model": "all", - "decoding": "greedy", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 320609, - "fn": 5791, - "accuracy": 0.9822579656862745 - }, - "0.01": { - "tp": 312156, - "fn": 14244, - "accuracy": 0.9563602941176471 - } - }, - "auroc": 0.977296466822406 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 202707, - "fn": 8493, - "accuracy": 0.9597869318181819 - }, - "0.01": { - "tp": 191571, - "fn": 19629, - "accuracy": 0.9070596590909091 - } - }, - "auroc": 0.9584006287237611 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 111805, - "fn": 3395, - "accuracy": 0.9705295138888889 - }, - "0.01": { - "tp": 106947, - "fn": 8253, - "accuracy": 0.928359375 - } - }, - "auroc": 0.9628022221317999 - }, - { - "domain": "all", - "model": "all", - "decoding": "sampling", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 314512, - "fn": 11888, - "accuracy": 0.9635784313725491 - }, - "0.01": { - "tp": 298518, - "fn": 27882, - "accuracy": 0.9145772058823529 - } - }, - "auroc": 0.9599541322795394 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "no", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 409075, - "fn": 13325, - "accuracy": 0.968454071969697 - }, - "0.01": { - "tp": 391190, - "fn": 31210, - "accuracy": 0.9261126893939394 - } - }, - "auroc": 0.9660526567555437 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "yes", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 226046, - "fn": 4354, - "accuracy": 0.9811024305555556 - }, - "0.01": { - "tp": 219484, - "fn": 10916, - "accuracy": 0.9526215277777778 - } - }, - "auroc": 0.9733418113425926 - }, - { - "domain": "all", - "model": "all", - "decoding": "all", - "repetition_penalty": "all", - "attack": "all", - "accuracy": { - "0.05": { - "tp": 635121, - "fn": 17679, - "accuracy": 0.9729181985294117 - }, - "0.01": { - "tp": 610674, - "fn": 42126, - "accuracy": 0.93546875 - } - }, - "auroc": 0.9686252995509728 - } - ], - "thresholds": { - "0.05": { - "abstracts": 0.8124823674268555, - "books": 0.999642793505227, - "news": 0.9995323266221932, - "poetry": 0.9990067337498294, - "recipes": 0.0031263396532814625, - "reddit": 0.9990944994539586, - "reviews": 0.99429978677902, - "wiki": 0.9986437261562422 - }, - "0.01": { - "abstracts": 0.9980292424268555, - "books": 0.9999327104974145, - "news": 0.9999137963487557, - "poetry": 0.9999222610935794, - "recipes": 0.9992200896532815, - "reddit": 0.9999184740633336, - "reviews": 0.99985398599777, - "wiki": 0.9998644292812422 - } - }, - "fpr": { - "0.05": { - "abstracts": 0.050000000000000044, - "books": 0.050000000000000044, - "news": 0.050000000000000044, - "poetry": 0.050000000000000044, - "recipes": 0.050000000000000044, - "reddit": 0.050000000000000044, - "reviews": 0.050000000000000044, - "wiki": 0.050000000000000044 - }, - "0.01": { - "abstracts": 0.010000000000000009, - "books": 0.010000000000000009, - "news": 0.010000000000000009, - "poetry": 0.010000000000000009, - "recipes": 0.010000000000000009, - "reddit": 0.010000000000000009, - "reviews": 0.010000000000000009, - "wiki": 0.010000000000000009 - } - } -} \ No newline at end of file