From f2879ce6ce80c1079c89df297b171e78ab9cada9 Mon Sep 17 00:00:00 2001 From: mengzhe gan Date: Tue, 17 Nov 2026 16:43:44 +0800 Subject: [PATCH] fix(utils): reject non-finite numbers in LLM JSON loaders Python's json module parses the NaN, Infinity and -Infinity literals that RFC 8259 does not define, so both repair_and_load_json and trim_and_load_json returned {'score': nan} for malformed model output and the nan silently poisoned every aggregate computed from it. #226/#232 already enforce this rule in the JSON graders; route both loaders through the shared reject_non_finite helper so structured-output parsing fails cleanly into its existing error handling instead. --- openjudge/utils/utils.py | 16 ++++++--- tests/utils/test_json_loaders.py | 61 ++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 tests/utils/test_json_loaders.py diff --git a/openjudge/utils/utils.py b/openjudge/utils/utils.py index 8de338a7e..9bf282272 100644 --- a/openjudge/utils/utils.py +++ b/openjudge/utils/utils.py @@ -12,6 +12,8 @@ from loguru import logger from pydantic import BaseModel +from openjudge.utils.json_utils import reject_non_finite + def repair_and_load_json( json_str: str, @@ -45,8 +47,12 @@ def repair_and_load_json( pass try: - return json.loads(repaired) - except json.JSONDecodeError as e: + # RFC 8259 defines no NaN/Infinity literals, but Python's json module + # parses them by default. Reject them so a malformed model response + # fails parsing (and hits the existing error handling) instead of + # feeding float('nan') into downstream scores and statistics. + return json.loads(repaired, parse_constant=reject_non_finite) + except ValueError as e: raise ValueError( f"Failed to decode JSON string `{json_str}` after repairing it " f"into `{repaired}`. Error: {e}", ) from e @@ -196,8 +202,10 @@ def trim_and_load_json(response: str, metric: Any = None) -> Dict[str, Any]: # Try to parse JSON try: - return json.loads(response) - except json.JSONDecodeError as e: + # Same RFC 8259 rule as repair_and_load_json: reject NaN/Infinity + # literals so they cannot flow into grader scores as float('nan'). + return json.loads(response, parse_constant=reject_non_finite) + except ValueError as e: error_msg = f"Failed to parse JSON from response: {e}\nResponse: {response[:200]}" if metric: metric_name = getattr(metric, "name", "unknown_metric") diff --git a/tests/utils/test_json_loaders.py b/tests/utils/test_json_loaders.py new file mode 100644 index 000000000..7c3ef7195 --- /dev/null +++ b/tests/utils/test_json_loaders.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Unit tests for the LLM-response JSON loaders (RFC 8259 enforcement).""" + +import pytest + +from openjudge.utils.utils import repair_and_load_json, trim_and_load_json + + +@pytest.mark.unit +class TestTrimAndLoadJson: + """Unit tests for trim_and_load_json""" + + def test_valid_json_parsed(self): + """Valid JSON passes through unchanged""" + assert trim_and_load_json('{"name": "Alice", "age": 30}') == {"name": "Alice", "age": 30} + + def test_code_fence_stripped(self): + """Markdown code fences around the JSON are removed before parsing""" + response = '```json\n{"score": 8, "reasoning": "Good"}\n```' + assert trim_and_load_json(response) == {"score": 8, "reasoning": "Good"} + + @pytest.mark.parametrize("literal", ["NaN", "Infinity", "-Infinity"]) + def test_non_finite_literal_rejected(self, literal: str): + """NaN/Infinity literals are invalid JSON and must fail parsing, not + feed float('nan')/float('inf') into downstream scores and statistics.""" + with pytest.raises(ValueError, match="not a valid JSON number"): + trim_and_load_json(f'{{"score": {literal}}}') + + def test_non_finite_literal_in_fenced_block_rejected(self): + """The rejection also applies to fenced LLM responses""" + with pytest.raises(ValueError, match="not a valid JSON number"): + trim_and_load_json('```json\n{"score": NaN}\n```') + + +@pytest.mark.unit +class TestRepairAndLoadJson: + """Unit tests for repair_and_load_json""" + + def test_valid_json_parsed(self): + """Valid JSON passes through unchanged""" + assert repair_and_load_json('{"key": "value"}') == {"key": "value"} + + def test_incomplete_json_repaired(self): + """Incomplete JSON is repaired (existing behavior, locked)""" + assert repair_and_load_json('{"a": "b"') == {"a": "b"} + + def test_complete_non_finite_literal_rejected(self): + """A complete NaN literal must fail parsing instead of loading as nan""" + with pytest.raises(ValueError, match="not a valid JSON number"): + repair_and_load_json('{"score": NaN}') + + def test_complete_infinity_literal_rejected(self): + """A complete Infinity literal must fail parsing instead of loading as inf""" + with pytest.raises(ValueError, match="not a valid JSON number"): + repair_and_load_json('{"score": Infinity}') + + def test_incomplete_non_finite_literal_repaired_to_string(self): + """json_repair stringifies a bare NaN while repairing incomplete input + (library behavior, locked so the grader never receives a float nan)""" + assert repair_and_load_json('{"score": NaN') == {"score": "NaN"}