Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions openjudge/utils/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
from loguru import logger
from pydantic import BaseModel

from openjudge.utils.json_utils import reject_non_finite


def repair_and_load_json(
json_str: str,
Expand Down Expand Up @@ -45,8 +47,12 @@ def repair_and_load_json(
pass

try:
return json.loads(repaired)
except json.JSONDecodeError as e:
# RFC 8259 defines no NaN/Infinity literals, but Python's json module
# parses them by default. Reject them so a malformed model response
# fails parsing (and hits the existing error handling) instead of
# feeding float('nan') into downstream scores and statistics.
return json.loads(repaired, parse_constant=reject_non_finite)
except ValueError as e:
raise ValueError(
f"Failed to decode JSON string `{json_str}` after repairing it " f"into `{repaired}`. Error: {e}",
) from e
Expand Down Expand Up @@ -196,8 +202,10 @@ def trim_and_load_json(response: str, metric: Any = None) -> Dict[str, Any]:

# Try to parse JSON
try:
return json.loads(response)
except json.JSONDecodeError as e:
# Same RFC 8259 rule as repair_and_load_json: reject NaN/Infinity
# literals so they cannot flow into grader scores as float('nan').
return json.loads(response, parse_constant=reject_non_finite)
except ValueError as e:
error_msg = f"Failed to parse JSON from response: {e}\nResponse: {response[:200]}"
if metric:
metric_name = getattr(metric, "name", "unknown_metric")
Expand Down
61 changes: 61 additions & 0 deletions tests/utils/test_json_loaders.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Unit tests for the LLM-response JSON loaders (RFC 8259 enforcement)."""

import pytest

from openjudge.utils.utils import repair_and_load_json, trim_and_load_json


@pytest.mark.unit
class TestTrimAndLoadJson:
"""Unit tests for trim_and_load_json"""

def test_valid_json_parsed(self):
"""Valid JSON passes through unchanged"""
assert trim_and_load_json('{"name": "Alice", "age": 30}') == {"name": "Alice", "age": 30}

def test_code_fence_stripped(self):
"""Markdown code fences around the JSON are removed before parsing"""
response = '```json\n{"score": 8, "reasoning": "Good"}\n```'
assert trim_and_load_json(response) == {"score": 8, "reasoning": "Good"}

@pytest.mark.parametrize("literal", ["NaN", "Infinity", "-Infinity"])
def test_non_finite_literal_rejected(self, literal: str):
"""NaN/Infinity literals are invalid JSON and must fail parsing, not
feed float('nan')/float('inf') into downstream scores and statistics."""
with pytest.raises(ValueError, match="not a valid JSON number"):
trim_and_load_json(f'{{"score": {literal}}}')

def test_non_finite_literal_in_fenced_block_rejected(self):
"""The rejection also applies to fenced LLM responses"""
with pytest.raises(ValueError, match="not a valid JSON number"):
trim_and_load_json('```json\n{"score": NaN}\n```')


@pytest.mark.unit
class TestRepairAndLoadJson:
"""Unit tests for repair_and_load_json"""

def test_valid_json_parsed(self):
"""Valid JSON passes through unchanged"""
assert repair_and_load_json('{"key": "value"}') == {"key": "value"}

def test_incomplete_json_repaired(self):
"""Incomplete JSON is repaired (existing behavior, locked)"""
assert repair_and_load_json('{"a": "b"') == {"a": "b"}

def test_complete_non_finite_literal_rejected(self):
"""A complete NaN literal must fail parsing instead of loading as nan"""
with pytest.raises(ValueError, match="not a valid JSON number"):
repair_and_load_json('{"score": NaN}')

def test_complete_infinity_literal_rejected(self):
"""A complete Infinity literal must fail parsing instead of loading as inf"""
with pytest.raises(ValueError, match="not a valid JSON number"):
repair_and_load_json('{"score": Infinity}')

def test_incomplete_non_finite_literal_repaired_to_string(self):
"""json_repair stringifies a bare NaN while repairing incomplete input
(library behavior, locked so the grader never receives a float nan)"""
assert repair_and_load_json('{"score": NaN') == {"score": "NaN"}
Loading