From ad7f61cc97065c50d11fe3143cc1ba589388b88f Mon Sep 17 00:00:00 2001 From: Abdul Rafey Date: Thu, 13 Aug 2026 23:40:43 +0500 Subject: [PATCH] fix(chat): stop refusing queries that ask to translate "translate" sat in the out-of-scope keyword list, so any query using the word without also hitting an in-scope keyword was hard-refused by the pre-filter and never reached the LLM scope classifier. The keyword layer cannot express the system prompt's "unrelated to course content" qualifier, so it over-refused: "translate this into Spanish" was rejected outright. The classifier still catches the genuine case, and "poem", "story", "fiction" and "rewrite" remain in the list. Adds the three previously-refused phrasings as regression cases, plus the research table's contrast case that already passed because an in-scope keyword short-circuits the check, and two non-English queries pinning the fall-through the pre-filter has always had but nothing asserted. Co-Authored-By: Claude Opus 5 (1M context) --- .../plugins/chat/assets/scope_keywords.json | 1 - .../chat/tests/test_scope_validation.py | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/sparkth/plugins/chat/assets/scope_keywords.json b/sparkth/plugins/chat/assets/scope_keywords.json index 8ccaadf5..a0cafa91 100644 --- a/sparkth/plugins/chat/assets/scope_keywords.json +++ b/sparkth/plugins/chat/assets/scope_keywords.json @@ -39,7 +39,6 @@ "can you code", "debug", "fix this", - "translate", "rewrite", "python", "javascript", diff --git a/sparkth/plugins/chat/tests/test_scope_validation.py b/sparkth/plugins/chat/tests/test_scope_validation.py index 686cc3a9..54bed34b 100644 --- a/sparkth/plugins/chat/tests/test_scope_validation.py +++ b/sparkth/plugins/chat/tests/test_scope_validation.py @@ -73,6 +73,25 @@ def test_course_related_query_with_potential_substring_match_is_in_scope(self) - # "curriculum" (in-scope) wins assert is_query_in_scope("Who is responsible for curriculum design in K-12?") is True + def test_translation_requests_reach_the_classifier(self) -> None: + """ "translate" must not hard-refuse: the keyword layer cannot express the + system prompt's "unrelated to course content" qualifier, so it over-refuses. + The LLM classifier downstream handles the genuinely out-of-scope case.""" + assert is_query_in_scope("translate this into Spanish") is True + assert is_query_in_scope("Can you translate my content to French") is True + assert is_query_in_scope("translate everything to German") is True + # Contrast case from the research table: this one already passed before the fix, + # because the in-scope "outline" short-circuits the out-of-scope check. Kept to + # pin that the in-scope override still wins. + assert is_query_in_scope("translate this outline into Spanish") is True + + def test_non_english_query_falls_through_to_the_classifier(self) -> None: + """A non-English query matches no English keyword either way, so the filter + returns True and the classifier decides. Pins existing behaviour that nothing + else asserts — the pre-filter is a fast path, never a non-English refusal.""" + assert is_query_in_scope("crea un curso sobre privacidad de datos") is True + assert is_query_in_scope("créer un cours sur la protection des données") is True + class TestStreamOutOfScopeRefusal: """Test the stream_out_of_scope_refusal SSE generator."""