From 6b182c8d9938c4bc7d338677553213914cc405f8 Mon Sep 17 00:00:00 2001 From: Noah Mathes Date: Wed, 12 Aug 2026 01:08:19 +0000 Subject: [PATCH] feat(scorers): pass config dictionary and runtime kwargs to pythonscorer script input Include config and kwargs dictionaries in the input JSON sent to PythonScorer scripts via stdin. Unpack extra config and kwargs keys top-level for direct script convenience. TAG=agy CONV=a8c331b9-fa2c-49e9-812d-ed07cd71d153 --- evalbench/scorers/pythonscorer.py | 11 ++++++++++- evalbench/test/pythonscorer_test.py | 27 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/evalbench/scorers/pythonscorer.py b/evalbench/scorers/pythonscorer.py index bc8efc17..b7616550 100644 --- a/evalbench/scorers/pythonscorer.py +++ b/evalbench/scorers/pythonscorer.py @@ -16,8 +16,9 @@ class PythonScorer(comparator.Comparator): def __init__(self, config: dict, name: str = "python_scorer"): super().__init__(config) + self.config = config or {} self.name = name - self.script_path = config.get("script_path") + self.script_path = self.config.get("script_path") if not self.script_path: raise ValueError("script_path is required for PythonScorer") @@ -59,7 +60,15 @@ def compare( "generated_error": generated_error, "database": database, "sqlite_db_dir": self.sqlite_db_dir, + "config": self.config, + "kwargs": kwargs, } + for k, v in self.config.items(): + if k not in input_data and k != "script_path": + input_data[k] = v + for k, v in kwargs.items(): + if k not in input_data: + input_data[k] = v try: json_input = json.dumps(input_data) diff --git a/evalbench/test/pythonscorer_test.py b/evalbench/test/pythonscorer_test.py index 6cd7f960..c033a157 100644 --- a/evalbench/test/pythonscorer_test.py +++ b/evalbench/test/pythonscorer_test.py @@ -87,6 +87,33 @@ def test_python_scorer_uv_not_found(self, mock_run): self.assertEqual(score, 0.0) self.assertIn("FAIL: 'uv' command not found", reason) + @patch('scorers.pythonscorer.subprocess.run') + def test_python_scorer_kwargs_and_config_passing(self, mock_run): + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"score": 100.0, "reason": "OK"}' + mock_result.stderr = "" + mock_run.return_value = mock_result + + config = {"script_path": "dummy_script.py", "custom_param": "value123"} + scorer = PythonScorer(config) + + scorer.compare( + nl_prompt="p", golden_query="g", query_type="DQL", + golden_execution_result="", golden_eval_result="", golden_error="", + generated_query="gen", generated_execution_result="", + generated_eval_result="", generated_error="", + extra_runtime_arg="arg456" + ) + + args, kwargs = mock_run.call_args + sent_input = json.loads(kwargs["input"]) + self.assertIn("kwargs", sent_input) + self.assertEqual(sent_input["kwargs"].get("extra_runtime_arg"), "arg456") + self.assertEqual(sent_input.get("extra_runtime_arg"), "arg456") + self.assertEqual(sent_input.get("custom_param"), "value123") + self.assertIn("config", sent_input) + if __name__ == '__main__': unittest.main()