Added GPTZero tests (and results) to ROC plot

Signed-off-by: Jacob Torrey <jacob@thinkst.com>

plot_rocs.py

@@ -9,7 +9,7 @@ from itertools import chain
 from math import sqrt
 from junitparser import JUnitXml
 
-MODELS = ['lzma', 'roberta']
+MODELS = ['lzma', 'roberta', 'gptzero']
 
 plt.figure()
 

test_gptzero_detect.py

@@ -15,13 +15,15 @@ human_files = os.listdir(HUMAN_SAMPLE_DIR)
 
 CONFIDENCE_THRESHOLD : float = 0.00 # What confidence to treat as error vs warning
 
-def test_training_file():
+def test_training_file(record_property):
     (classification, score) = run_on_file_chunked('ai-generated.txt')
+    record_property("score", str(score))
     assert classification == 'AI', 'The training corpus should always be detected as AI-generated... since it is (score: ' + str(round(score, 8)) + ')'
 
 @pytest.mark.parametrize('f', human_files)
-def test_human_samples(f):
+def test_human_samples(f, record_property):
     (classification, score) = run_on_file_chunked(HUMAN_SAMPLE_DIR + f)
+    record_property("score", str(score))
     if score > CONFIDENCE_THRESHOLD:
         assert classification == 'Human', f + ' is a human-generated file, misclassified as AI-generated with confidence ' + str(round(score, 8))
     else:
@@ -31,8 +33,9 @@ def test_human_samples(f):
             warn("Unable to confidently classify: " + f)
 
 @pytest.mark.parametrize('f', ai_files)
-def test_llm_sample(f):
+def test_llm_sample(f, record_property):
    (classification, score) = run_on_file_chunked(AI_SAMPLE_DIR + f)
+   record_property("score", str(score))
    if score > CONFIDENCE_THRESHOLD:
        assert classification == 'AI', f + ' is an LLM-generated file, misclassified as human-generated with confidence ' + str(round(score, 8))
    else:
@@ -48,8 +51,9 @@ with jsonlines.open(HUMAN_JSONL_FILE) as reader:
         human_samples.append(obj)
 
 @pytest.mark.parametrize('i', human_samples[0:NUM_JSONL_SAMPLES])
-def test_human_jsonl(i):
+def test_human_jsonl(i, record_property):
     (classification, score) = run_on_text_chunked(i.get('text', ''))
+    record_property("score", str(score))
     assert classification == 'Human', HUMAN_JSONL_FILE + ':' + str(i.get('id')) + ' (len: ' + str(i.get('length', -1)) + ') is a human-generated sample, misclassified as AI-generated with confidence ' + str(round(score, 8))
 
 AI_JSONL_FILE = 'samples/xl-1542M.test.jsonl'
@@ -59,8 +63,9 @@ with jsonlines.open(AI_JSONL_FILE) as reader:
         ai_samples.append(obj)
 
 @pytest.mark.parametrize('i', ai_samples[0:NUM_JSONL_SAMPLES])
-def test_llm_jsonl(i):
+def test_llm_jsonl(i, record_property):
     (classification, score) = run_on_text_chunked(i.get('text', ''))
+    record_property("score", str(score))
     assert classification == 'AI', AI_JSONL_FILE + ':' + str(i.get('id')) + ' (text: ' + i.get('text', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))
 
 GPT3_JSONL_FILE = 'samples/GPT-3-175b_samples.jsonl'
@@ -72,8 +77,9 @@ with jsonlines.open(GPT3_JSONL_FILE) as reader:
                 gpt3_samples.append(l)
 
 @pytest.mark.parametrize('i', gpt3_samples[0:NUM_JSONL_SAMPLES])
-def test_gpt3_jsonl(i):
+def test_gpt3_jsonl(i, record_property):
     (classification, score) = run_on_text_chunked(i)
+    record_property("score", str(score))
     assert classification == 'AI', GPT3_JSONL_FILE + ' is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))
 
 NEWS_JSONL_FILE = 'samples/news.jsonl'
@@ -83,11 +89,13 @@ with jsonlines.open(NEWS_JSONL_FILE) as reader:
         news_samples.append(obj)
 
 @pytest.mark.parametrize('i', news_samples[0:NUM_JSONL_SAMPLES])
-def test_humannews_jsonl(i):
+def test_humannews_jsonl(i, record_property):
     (classification, score) = run_on_text_chunked(i.get('human', ''))
+    record_property("score", str(score))
     assert classification == 'Human', NEWS_JSONL_FILE + ' is a human-generated sample, misclassified as AI-generated with confidence ' + str(round(score, 8))
 
 @pytest.mark.parametrize('i', news_samples[0:NUM_JSONL_SAMPLES])
-def test_chatgptnews_jsonl(i):
+def test_chatgptnews_jsonl(i, record_property):
     (classification, score) = run_on_text_chunked(i.get('chatgpt', ''))
+    record_property("score", str(score))
     assert classification == 'AI', NEWS_JSONL_FILE + ' is a AI-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))