diff --git a/contentatscale-report.xml b/contentatscale-report.xml new file mode 100644 index 0000000..2fe53d6 --- /dev/null +++ b/contentatscale-report.xml @@ -0,0 +1,235 @@ +E AssertionError: samples/webtext.test.jsonl:255006 (len: 365) is a human-generated sample, misclassified as AI-generated with confidence 0.86 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/webtext.test.jsonl:255016 (len: 92) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/webtext.test.jsonl:255067 (len: 847) is a human-generated sample, misclassified as AI-generated with confidence 0.6 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.56 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.88 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.54 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.6 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.92 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-init.jsonl:8600008 [885] (title: Vertical Handof) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600018 [642] (title: Efficient Detec) is a human-generated sample, misclassified as AI-generated with confidence 0.97 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600029 [1139] (title: A Social Bots D) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600048 [820] (title: Improved Data-A) is a human-generated sample, misclassified as AI-generated with confidence 0.98 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600060 [731] (title: Efficient Chann) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600062 [699] (title: A Novel Route R) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600074 [1015] (title: Human Activity ) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600085 [855] (title: The Residual An) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600101 [470] (title: Closed-form App) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600111 [489] (title: A GMM-UBM Based) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600114 [590] (title: An Analysis of ) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600150 [867] (title: User Associatio) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600154 [719] (title: Prediction of H) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600210 [1049] (title: A Fingerprint L) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600225 [969] (title: Quantum Key Int) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600228 [473] (title: TMVM: Truncated) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600233 [961] (title: Power System Fr) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600234 [880] (title: Association Ana) is a human-generated sample, misclassified as AI-generated with confidence 0.86 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600013 (title: Robust offline trained neural network for TDOA bas) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600014 (title: Gaussian MAC with Feedback and Strictly Causal Sta) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600045 (title: A Method Based on Frequent Pattern Mining to Predi) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE httpcore.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None> + +The above exception was the direct cause of the following exception: +E httpx.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None>E AssertionError: samples/ieee-chatgpt-polish.jsonl:8600080 (title: Identification of Coronary Artery Diseased Subject) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600088 (title: Weighted Nuclear Norm and TV Regularization based ) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE httpcore.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None> + +The above exception was the direct cause of the following exception: +E httpx.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None>E AssertionError: samples/ieee-chatgpt-polish.jsonl:8600091 (title: Improved Epoch Extraction Using Variational Mode D) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600094 (title: NLOS Detection and Mitigation for UWB/IMU Fusion S) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600099 (title: Ranking-based Collaborative Clustering for Heterog) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600131 (title: Cell-Phone Identification from Recompressed Audio ) is an LLM-generated sample, misclassified as human-generated with confidence 0.95 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600132 (title: Spatial Image Steganalysis Based on ResNeXt) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600143 (title: Balanced Use of Battery Power in Ad-hoc Wireless S) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600153 (title: A New Combinatorial Design Based Data En-Route Fil) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600169 (title: LayerOS: Scheduling Apps between the Local System ) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600178 (title: Cost Effective Hash Chain Based Key Pre-Distributi) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600186 (title: Atmospheric Absorption Loss Estimation of Terahert) is an LLM-generated sample, misclassified as human-generated with confidence 0.63 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600190 (title: Manner of Articulation based Split Lattices for Ph) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600201 (title: Control and Management of Optical Networks Using O) is an LLM-generated sample, misclassified as human-generated with confidence 0.98 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600202 (title: Mridangam Artist Identification from Taniavartanam) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600206 (title: A 3D Placement of Unmanned Aerial Vehicle Base Sta) is an LLM-generated sample, misclassified as human-generated with confidence 0.97 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600219 (title: Deep Learning Intrusion Detection Model Based on O) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600231 (title: Uplink Asynchronous Fractional Pilots Scheduling i) is an LLM-generated sample, misclassified as human-generated with confidence 0.84 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-vicuna-generation.jsonl:8600178 (title: Cost Effective Hash Chain Based Key Pre-Distributi) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE httpcore.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None> + +The above exception was the direct cause of the following exception: +E httpx.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None>E AssertionError: samples/gptzero_eval.csv:4 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE httpcore.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None> + +The above exception was the direct cause of the following exception: +E httpx.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None>E AssertionError: samples/gptzero_eval.csv:7 was misclassified with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:10 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE httpcore.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None> + +The above exception was the direct cause of the following exception: +E httpx.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None>E AssertionError: samples/gptzero_eval.csv:14 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:19 was misclassified with confidence 0.59 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:21 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:22 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:24 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:25 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:26 was misclassified with confidence 0.97 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:27 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:30 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:31 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:32 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:34 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:35 was misclassified with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:36 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:40 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:41 was misclassified with confidence 0.93 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:51 was misclassified with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/gptzero_eval.csv:75 was misclassified with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AI \ No newline at end of file diff --git a/contentatscale_detect.py b/contentatscale_detect.py index d095682..b566458 100644 --- a/contentatscale_detect.py +++ b/contentatscale_detect.py @@ -17,7 +17,7 @@ def make_req(text : str) -> Optional[str]: 'Referer': 'https://contentatscale.ai/ai-content-detector/' } data = 'content=' + urllib.parse.quote_plus(text) + '&action=checkaiscore' - c = httpx.Client(http2=True, timeout=30.0) + c = httpx.Client(http2=True, timeout=60.0) res = c.post(API_URL, headers=headers, data=data) if res.status_code != 200: print(res.text) @@ -28,7 +28,7 @@ def make_req(text : str) -> Optional[str]: return res.json().get('score') def classify_text(s : str) -> Optional[Tuple[str, float]]: - res = int(make_req(s)) / 100 + res = int(make_req(s)) / 100.0 if res is None: print("Unable to classify!") return None @@ -39,9 +39,9 @@ def classify_text(s : str) -> Optional[Tuple[str, float]]: except TypeError as e: print("Unable to convert " + str(res) + " to float!") if res < 0.5: - return ('AI', res) + return ('AI', 1 - res) else: - return ('Human', 1 - res) + return ('Human', res) def run_on_file_chunked(filename : str, chunk_size : int = 3000) -> Optional[Tuple[str, float]]: ''' diff --git a/plot_rocs.py b/plot_rocs.py index 908014a..e5ca647 100644 --- a/plot_rocs.py +++ b/plot_rocs.py @@ -6,7 +6,7 @@ from sklearn.metrics import roc_curve, auc import re from junitparser import JUnitXml -MODELS = ['zippy', 'roberta', 'gptzero', 'openai', 'crossplag', 'contentatscale'] +MODELS = ['zippy', 'roberta', 'gptzero', 'crossplag', 'contentatscale'] SKIPCASES = ['gpt2', 'gpt3'] MAX_PER_CASE = 500