From 9f5743e7321b61c575a989bcaeb987d61a00c5d2 Mon Sep 17 00:00:00 2001 From: rbro112 Date: Fri, 11 Sep 2026 17:21:06 -0700 Subject: [PATCH] Add some new autofix_agent tests that act more as integration tests --- .../sentry/seer/autofix/test_autofix_agent.py | 294 ++++++++++++++++++ 1 file changed, 294 insertions(+) diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index 32d88e5432fd..c0508ad3b20b 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -1,3 +1,4 @@ +from typing import Any from unittest.mock import MagicMock, patch import pytest @@ -31,11 +32,22 @@ from sentry.seer.autofix.commit_author import SeerCommitAuthor from sentry.seer.autofix.constants import AutofixReferrer from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.feature.models import FEATURE_ID +from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import AutofixStoppingPoint from sentry.seer.models import SeerPermissionError +from sentry.seer.models.run import ( + SeerAgentRun, + SeerRun, + SeerRunMilestone, + SeerRunMilestoneType, + SeerRunMirrorStatus, + SeerRunType, +) from sentry.sentry_apps.utils.webhooks import SeerActionType from sentry.testutils.cases import TestCase +from sentry.testutils.helpers.task_runner import BurstTaskRunner from sentry.types.activity import ActivityType from sentry.utils import json @@ -56,6 +68,52 @@ def _make_scm_mock(*, get_repository=None, get_branch=None): )() +class _FakeSeerResponse: + def __init__(self, data: dict[str, Any], status: int = 200) -> None: + self.data = data + self.status = status + + def json(self) -> dict[str, Any]: + return self.data + + +class _RecordingSeer: + """Record requests at the Seer transport boundary and serve fake run state.""" + + def __init__(self, run_id: int = 4242) -> None: + self.run_id = run_id + self.feature_requests: list[dict[str, Any]] = [] + self.chat_requests: list[dict[str, Any]] = [] + self.state_requests: list[dict[str, Any]] = [] + self.run_state: SeerRunState | None = None + + def start_feature_run(self, body: dict[str, Any], **_kwargs: Any) -> _FakeSeerResponse: + self.feature_requests.append(body) + return _FakeSeerResponse({"run_id": self.run_id}) + + def continue_run(self, body: dict[str, Any], **_kwargs: Any) -> _FakeSeerResponse: + self.chat_requests.append(body) + return _FakeSeerResponse({}) + + def fetch_run_state(self, body: dict[str, Any], **_kwargs: Any) -> _FakeSeerResponse: + self.state_requests.append(body) + assert self.run_state is not None, "A fake Seer run state must be configured" + session = self.run_state.dict() + # SeerRunState excludes internal metadata from public serialization, but + # Seer's internal state endpoint includes it for trusted Sentry callers. + session["metadata"] = self.run_state.metadata + return _FakeSeerResponse({"session": session}) + + +class _FakeSCM: + def get_repository(self) -> dict[str, Any]: + return {"data": {"default_branch": "main"}} + + def get_branch(self, branch: str) -> dict[str, Any]: + assert branch == "main" + return {"data": {"sha": "abc123"}} + + class TestGenerateAutofixHandoffPrompt(TestCase): """Tests for generate_autofix_handoff_prompt function.""" @@ -458,6 +516,242 @@ def test_pr_iteration_prompt_without_run_state_omits_pr_links(self) -> None: assert "pull request(s)" not in prompt +class TestAutofixAgentIntegration(TestCase): + """Exercise Autofix orchestration with real Sentry state and a fake Seer boundary.""" + + def setUp(self) -> None: + super().setUp() + self.group = self.create_group( + project=self.project, + message="TypeError: a request without a user reached the task handler", + ) + self.group.culprit = "app.tasks.run" + self.group.save(update_fields=["culprit"]) + repository = self.create_repo( + project=self.project, + provider="integrations:github", + external_id="123", + name="test-org/test-repo", + ) + self.create_seer_project_repository(project=self.project, repository=repository) + + def _completed_root_cause_state(self, run_id: int) -> SeerRunState: + return SeerRunState( + run_id=run_id, + blocks=[ + MemoryBlock( + id="root-cause-block", + message=Message( + role="assistant", + content="The task assumes every request has a user.", + metadata={ + "step": AutofixStep.ROOT_CAUSE.value, + "referrer": AutofixReferrer.WEB.value, + }, + ), + timestamp="2024-01-01T00:00:00Z", + artifacts=[ + Artifact( + key="root_cause", + data={ + "headline": "Task accepts a missing user", + "one_line_description": ( + "A missing user reaches a task that assumes one exists." + ), + "five_whys": ["The request did not include a user."], + "reproduction_steps": ["Submit the task without a user."], + "relevant_repo": "test-org/test-repo", + "fixability": { + "assessment": "fixable", + "reason": "The task can validate its input.", + }, + }, + reason="Generated by the Autofix agent", + ) + ], + ) + ], + status="completed", + updated_at="2024-01-01T00:00:00Z", + metadata={"group_id": self.group.id}, + ) + + def test_root_cause_step_persists_and_dispatches_feature_run(self) -> None: + seer = _RecordingSeer() + scm = _FakeSCM() + + with ( + self.feature( + { + "organizations:gen-ai-features": True, + "organizations:autofix-rca-in-seer": True, + } + ), + patch("sentry.scm.factory.new", return_value=scm), + patch( + "sentry.receivers.outbox.cell.make_feature_run_request", + side_effect=seer.start_feature_run, + ), + BurstTaskRunner() as tasks, + ): + run = trigger_autofix_agent( + group=self.group, + step=AutofixStep.ROOT_CAUSE, + referrer=AutofixReferrer.WEB, + stopping_point=AutofixStoppingPoint.ROOT_CAUSE, + user_context="Requests without a user fail.", + user=self.user, + ) + + run.refresh_from_db() + assert run.seer_run_state_id == seer.run_id + assert run.mirror_status == SeerRunMirrorStatus.LIVE + assert run.type == SeerRunType.FEATURE_RUN + assert run.user_id == self.user.id + + agent_run = SeerAgentRun.objects.get(run=run) + assert agent_run.source == FEATURE_ID + assert agent_run.project_id == self.project.id + assert agent_run.group_id == self.group.id + assert agent_run.extras == { + "referrer": AutofixReferrer.WEB.value, + "stopping_point": AutofixStoppingPoint.ROOT_CAUSE.value, + } + + assert len(seer.feature_requests) == 1 + request = seer.feature_requests[0] + assert request["feature_id"] == FEATURE_ID + assert request["ref"] == str(run.uuid) + assert request["external_idempotency_key"] == str(run.uuid) + payload = request["payload"] + assert payload["group_id"] == self.group.id + assert payload["project_id"] == self.project.id + assert payload["short_id"] == self.group.qualified_short_id + assert payload["title"] == self.group.title + assert payload["culprit"] == "app.tasks.run" + assert payload["user_context"] == "Requests without a user fail." + assert payload["stopping_point"] == AutofixStoppingPoint.ROOT_CAUSE.value + assert payload["repo_pins"] == { + "test-org/test-repo": { + "sha": "abc123", + "branch": "main", + "base_sha": "abc123", + "base_branch": "main", + } + } + + assert Activity.objects.filter( + group=self.group, + type=ActivityType.SEER_RCA_STARTED.value, + data={"run_id": seer.run_id}, + ).exists() + assert [ + kwargs["event_name"] + for task, _args, kwargs in tasks.queue + if task.name + == "sentry.sentry_apps.tasks.sentry_apps.broadcast_webhooks_for_organization" + ] == [SeerActionType.ROOT_CAUSE_STARTED.value] + + def test_completed_root_cause_continues_same_run_with_solution_prompt(self) -> None: + seer = _RecordingSeer() + scm = _FakeSCM() + + with ( + self.feature( + { + "organizations:gen-ai-features": True, + "organizations:autofix-rca-in-seer": True, + } + ), + patch("sentry.scm.factory.new", return_value=scm), + patch( + "sentry.receivers.outbox.cell.make_feature_run_request", + side_effect=seer.start_feature_run, + ), + patch( + "sentry.seer.agent.client_utils.make_agent_state_request", + side_effect=seer.fetch_run_state, + ), + patch( + "sentry.seer.agent.client.make_agent_chat_request", + side_effect=seer.continue_run, + ), + BurstTaskRunner() as tasks, + ): + run = trigger_autofix_agent( + group=self.group, + step=AutofixStep.ROOT_CAUSE, + referrer=AutofixReferrer.WEB, + stopping_point=AutofixStoppingPoint.SOLUTION, + user=self.user, + ) + seer.run_state = self._completed_root_cause_state(seer.run_id) + + AutofixOnCompletionHook.execute(self.organization, seer.run_id) + + assert len(seer.state_requests) == 2 + assert all(request["run_id"] == seer.run_id for request in seer.state_requests) + assert len(seer.chat_requests) == 1 + solution_request = seer.chat_requests[0] + assert solution_request["run_id"] == seer.run_id + assert self.group.qualified_short_id in solution_request["query"] + assert self.group.title in solution_request["query"] + assert "Do NOT implement" in solution_request["query"] + assert solution_request["artifact_key"] == AutofixStep.SOLUTION.value + assert set(solution_request["artifact_schema"]["required"]) == { + "one_line_summary", + "steps", + } + assert solution_request["query_metadata"] == { + "step": AutofixStep.SOLUTION.value, + "referrer": AutofixReferrer.WEB.value, + "has_user_context": "no", + "is_retry": "no", + } + + assert SeerRun.objects.filter(organization=self.organization).count() == 1 + run.refresh_from_db() + assert run.seer_run_state_id == seer.run_id + + milestone = SeerRunMilestone.objects.get( + seer_run=run, + milestone=SeerRunMilestoneType.ROOT_CAUSE, + ) + assert milestone.extras == { + "root_cause_artifact": { + "headline": "Task accepts a missing user", + "one_line_description": "A missing user reaches a task that assumes one exists.", + } + } + + completed_activity = Activity.objects.get( + group=self.group, + type=ActivityType.SEER_RCA_COMPLETED.value, + ) + assert completed_activity.data == { + "run_id": seer.run_id, + "summary": "A missing user reaches a task that assumes one exists.", + } + assert Activity.objects.filter( + group=self.group, + type=ActivityType.SEER_SOLUTION_STARTED.value, + data={"run_id": seer.run_id}, + ).exists() + + self.group.refresh_from_db() + assert self.group.seer_explorer_autofix_last_triggered is not None + assert [ + kwargs["event_name"] + for task, _args, kwargs in tasks.queue + if task.name + == "sentry.sentry_apps.tasks.sentry_apps.broadcast_webhooks_for_organization" + ] == [ + SeerActionType.ROOT_CAUSE_STARTED.value, + SeerActionType.ROOT_CAUSE_COMPLETED.value, + SeerActionType.SOLUTION_STARTED.value, + ] + + class TestTriggerAutofixAgent(TestCase): def setUp(self) -> None: super().setUp()