diff --git a/backend/kernelCI_app/constants/localization.py b/backend/kernelCI_app/constants/localization.py index 0d3d90f03..d63dd24fc 100644 --- a/backend/kernelCI_app/constants/localization.py +++ b/backend/kernelCI_app/constants/localization.py @@ -34,6 +34,7 @@ class ClientStrings: ISSUE_NOT_FOUND = "Issue not found" NO_ISSUE_FOUND = "No issues found" INVALID_JSON_BODY = "Invalid body, request body must be a valid json string" + INVALID_FILTERS = "Invalid filter key or value" ISSUE_EMPTY_LIST = "Invalid body, the issue list must not be empty" ISSUE_NO_EXTRA_DETAILS = ( "No extra details found. Issue id has no incident or doesn't exist." diff --git a/backend/kernelCI_app/helpers/filters.py b/backend/kernelCI_app/helpers/filters.py index 76f1cef84..17d673d8c 100644 --- a/backend/kernelCI_app/helpers/filters.py +++ b/backend/kernelCI_app/helpers/filters.py @@ -5,6 +5,7 @@ from kernelCI_app.constants.general import UNCATEGORIZED_STRING from kernelCI_app.helpers.commonDetails import PossibleTabs from kernelCI_app.helpers.logger import log_message +from kernelCI_app.models import StatusChoices from kernelCI_app.typeModels.databases import ( StatusValues, failure_status_list, @@ -22,6 +23,10 @@ NULL_STRINGS = set(["null", UNKNOWN_STRING, "NULL"]) +def is_valid_status(status: str) -> bool: + return status.upper() in {*StatusChoices, "NULL"} + + def is_status_failure( test_status: StatusValues, fail_list: list[StatusValues] = failure_status_list ) -> bool: @@ -129,6 +134,10 @@ def is_issue_filtered_out( return not in_filter +def is_filtered_out(value: str, filter_values: set[str]): + return filter_values and value not in filter_values + + def should_filter_test_issue( *, issue_filters: set, @@ -361,6 +370,7 @@ def __init__(self, data: Dict, process_body=False) -> None: "test.status": self._handle_test_status, "test.duration": self._handle_test_duration, "build.status": self._handle_build_status, + "duration": self._handle_build_duration, # TODO: same as build.duration (should be standardized) "build.duration": self._handle_build_duration, "origin": self._handle_origins, "config_name": self._handle_config_name, @@ -391,6 +401,14 @@ def __init__(self, data: Dict, process_body=False) -> None: self._process_filters() + def __repr__(self) -> str: + parts = "" + for parsed_filter in self.filters: + parts += "\n\t{},".format( + ", ".join([f"{key}={val}" for key, val in parsed_filter.items()]) + ) + return f"FilterParams({parts})" + def _handle_boot_status(self, current_filter: ParsedFilter) -> None: self.filterBootStatus.add(current_filter["value"]) diff --git a/backend/kernelCI_app/helpers/hardwareDetails.py b/backend/kernelCI_app/helpers/hardwareDetails.py index 0001832b5..67af2c20b 100644 --- a/backend/kernelCI_app/helpers/hardwareDetails.py +++ b/backend/kernelCI_app/helpers/hardwareDetails.py @@ -509,8 +509,8 @@ def process_issue( incident_test_id=record["incidents__test_id"], build_status=record["build__status"], test_status=record["status"], - issue_comment=record["incidents__issue__comment"], - issue_report_url=record["incidents__issue__report_url"], + issue_comment=record.get("incidents__issue__comment"), + issue_report_url=record.get("incidents__issue__report_url"), is_failed_task=is_failed_task, issue_from=issue_from, task=task_issues_dict, diff --git a/backend/kernelCI_app/helpers/issueExtras.py b/backend/kernelCI_app/helpers/issueExtras.py index fd61119f4..3fc4da5c4 100644 --- a/backend/kernelCI_app/helpers/issueExtras.py +++ b/backend/kernelCI_app/helpers/issueExtras.py @@ -1,6 +1,7 @@ from collections import defaultdict -from typing import List, Tuple +from typing import List, Tuple, Optional +from kernelCI_app.constants.general import UNCATEGORIZED_STRING from kernelCI_app.helpers.logger import log_message from kernelCI_app.queries.issues import get_issue_first_seen_data, get_issue_trees_data from kernelCI_app.typeModels.issues import ( @@ -20,6 +21,14 @@ class TagUrls: ) +def parse_issue(issue_str: Optional[str]) -> tuple[str, Optional[int]]: + if issue_str is None: + return (UNCATEGORIZED_STRING, None) + issue_id, _, issue_version = issue_str.partition(",") + issue_version = int(issue_version) if issue_version.upper() != "NULL" else None + return (issue_id, issue_version) + + def process_issues_extra_details( *, issue_key_list: List[Tuple[str, int]], diff --git a/backend/kernelCI_app/queries/hardware.py b/backend/kernelCI_app/queries/hardware.py index b1adfb9ff..4360c312f 100644 --- a/backend/kernelCI_app/queries/hardware.py +++ b/backend/kernelCI_app/queries/hardware.py @@ -1,4 +1,4 @@ -from typing import TypedDict +from typing import Optional, TypedDict from datetime import datetime from django.db import connection @@ -339,15 +339,220 @@ def get_hardware_details_data( return records +def _get_build_duration_clause( + builds_duration: tuple[Optional[int], Optional[int]] +) -> str: + clause = "" + + # builds + duration_min, duration_max = builds_duration + if duration_min: + clause += "AND builds.duration >= %(build_duration_min)s\n" + if duration_max: + clause += "AND builds.duration <= %(build_duration_max)s\n" + + return clause + + +def _get_boot_test_duration_clause( + boots_duration: tuple[Optional[int], Optional[int]], + tests_duration: tuple[Optional[int], Optional[int]], +) -> str: + clause = "" + + # tests + duration_min, duration_max = tests_duration + if duration_min: + clause += ( + "AND ((tests.path like 'boot.%%' or tests.path = 'boot') " + "OR tests.duration >= %(test_duration_min)s)\n" + ) + if duration_max: + clause += ( + "AND ((tests.path like 'boot.%%' or tests.path = 'boot') " + "OR tests.duration <= %(test_duration_max)s)\n" + ) + + # boots + duration_min, duration_max = boots_duration + if duration_min: + clause += ( + "AND (NOT (tests.path like 'boot.%%' or tests.path = 'boot') " + "OR tests.duration >= %(boot_duration_min)s)\n" + ) + if duration_max: + clause += ( + "AND (NOT (tests.path like 'boot.%%' or tests.path = 'boot') " + "OR tests.duration <= %(boot_duration_max)s)\n" + ) + + return clause + + +def get_hardware_details_summary( + *, + hardware_id: str, + origin: str, + commit_hashes: list[str], + builds_duration: Optional[tuple[Optional[int], Optional[int]]] = None, + boots_duration: Optional[tuple[Optional[int], Optional[int]]] = None, + tests_duration: Optional[tuple[Optional[int], Optional[int]]] = None, + start_datetime: datetime, + end_datetime: datetime, +): + + if builds_duration is None: + builds_duration = (None, None) + if boots_duration is None: + boots_duration = (None, None) + if tests_duration is None: + tests_duration = (None, None) + + cache_key = "hardwareDetailsSummary" + + tests_cache_params = { + "hardware_id": hardware_id, + "origin": origin, + "commit_hashes": commit_hashes, + "start_date": start_datetime.timestamp(), + "end_date": end_datetime.timestamp(), + "builds_duration": builds_duration, + "boots_duration": boots_duration, + "tests_duration": tests_duration, + } + + query_rows = get_query_cache(cache_key, tests_cache_params) + + if query_rows is not None: + return query_rows + + builds_duration_clause = _get_build_duration_clause(builds_duration) + boots_tests_duration_clause = _get_boot_test_duration_clause( + boots_duration, tests_duration + ) + + query = """ + (SELECT + COUNT(DISTINCT builds.id) AS count, + checkouts.origin, + builds.status AS status, + count(DISTINCT incidents.id) AS incidents_count, + array_agg(DISTINCT incidents.issue_id || ',' || incidents.issue_version::text) + AS known_issues, + array[builds.compiler, builds.architecture] AS compiler_arch, + builds.config_name, + builds.misc->>'lab' AS lab, + tests.environment_misc->>'platform' AS platform, + tests.environment_compatible, + checkouts.origin, + checkouts.tree_name, + checkouts.git_repository_url, + checkouts.git_commit_tags, + checkouts.git_commit_name, + checkouts.git_repository_branch, + checkouts.git_commit_hash, + true AS is_build, + false AS is_test, + false AS is_boot + FROM + builds + INNER JOIN tests ON + tests.build_id = builds.id + INNER JOIN checkouts ON + builds.checkout_id = checkouts.id + LEFT OUTER JOIN incidents ON + builds.id = incidents.build_id + WHERE + ( + builds.config_name IS NOT NULL + AND builds.id not like 'maestro:dummy_%%' + AND (tests.environment_compatible @> ARRAY[%(platform)s]::TEXT[] + OR tests.environment_misc ->> 'platform' = %(platform)s) + ) + AND builds.origin = %(origin)s + AND builds.start_time >= %(start_date)s + AND builds.start_time <= %(end_date)s + AND (checkouts.git_commit_hash = ANY(%(commits)s)) {0} + GROUP BY checkouts.id, builds.status, tests.environment_compatible, compiler_arch, + builds.config_name, lab, platform, is_boot) + UNION ALL + (SELECT + COUNT(*) AS count, + checkouts.origin, + tests.status AS status, + count(DISTINCT incidents.id) as incidents_count, + array_agg(DISTINCT incidents.issue_id || ',' || incidents.issue_version::text) + as known_issues, + array[builds.compiler, builds.architecture] AS compiler_arch, + builds.config_name, + tests.misc->>'runtime' AS lab, + tests.environment_misc->>'platform' AS platform, + tests.environment_compatible, + checkouts.origin, + checkouts.tree_name, + checkouts.git_repository_url, + checkouts.git_commit_tags, + checkouts.git_commit_name, + checkouts.git_repository_branch, + checkouts.git_commit_hash, + false AS is_build, + true AS is_test, + (tests.path like 'boot.%%' or tests.path = 'boot') AS is_boot + FROM + builds + INNER JOIN tests ON + tests.build_id = builds.id + INNER JOIN checkouts ON + builds.checkout_id = checkouts.id + LEFT OUTER JOIN incidents ON + tests.id = incidents.test_id + WHERE + ( + (tests.environment_compatible @> ARRAY[%(platform)s]::TEXT[] + OR tests.environment_misc ->> 'platform' = %(platform)s) + ) + AND tests.origin = %(origin)s + AND tests.start_time >= %(start_date)s + AND tests.start_time <= %(end_date)s + AND (checkouts.git_commit_hash = ANY(%(commits)s)) {1} + GROUP BY checkouts.id, tests.status, tests.environment_compatible, compiler_arch, + builds.config_name, lab, platform, is_boot); + """.format( + builds_duration_clause, + boots_tests_duration_clause, + ) + + build_duration_min, build_duration_max = builds_duration + boot_duration_min, boot_duration_max = boots_duration + test_duration_min, test_duration_max = tests_duration + + params = { + "platform": hardware_id, + "origin": origin, + "start_date": start_datetime, + "end_date": end_datetime, + "commits": commit_hashes, + "build_duration_min": build_duration_min, + "build_duration_max": build_duration_max, + "boot_duration_min": boot_duration_min, + "boot_duration_max": boot_duration_max, + "test_duration_min": test_duration_min, + "test_duration_max": test_duration_max, + } + + with connection.cursor() as cursor: + cursor.execute(query, params) + query_rows = dict_fetchall(cursor) + set_query_cache(key=cache_key, params=tests_cache_params, rows=query_rows) + return query_rows + + def query_records( *, hardware_id: str, origin: str, trees: list[Tree], start_date: int, end_date: int ) -> list[dict] | None: commit_hashes = [tree.head_git_commit_hash for tree in trees] - # TODO Treat commit_hash collision (it can happen between repos) - with connection.cursor() as cursor: - cursor.execute( - """ + query = """ SELECT tests.id, tests.origin AS test_origin, @@ -411,19 +616,22 @@ def query_records( ORDER BY issues."_timestamp" DESC """.format( - ",".join(["%s"] * len(commit_hashes)) - ), - [ - hardware_id, - hardware_id, - origin, - start_date, - end_date, - ] - + commit_hashes, - ) + ",".join(["%s"] * len(commit_hashes)) + ) - return dict_fetchall(cursor) + params = [ + hardware_id, + hardware_id, + origin, + start_date, + end_date, + ] + commit_hashes + + # TODO Treat commit_hash collision (it can happen between repos) + with connection.cursor() as cursor: + cursor.execute(query, params) + query_rows = dict_fetchall(cursor) + return query_rows def get_hardware_summary_data( @@ -512,6 +720,90 @@ def get_hardware_summary_data( return dict_fetchall(cursor) +def get_hardware_trees_head_commits( + *, + hardware_id: str, + origin: str, + start_datetime: datetime, + end_datetime: datetime, +) -> list[tuple[str, str]]: + + # similar to the get_hardware_trees_data, except we limit the information + # being brought to the commit hash + + cache_key = "hardwareTreesHeadCommits" + + cache_params = { + "hardware": hardware_id, + "origin": origin, + "start_date": start_datetime.timestamp(), + "end_date": end_datetime.timestamp(), + } + + trees: list[tuple[str, str]] = get_query_cache(cache_key, cache_params) + + if trees: + return trees + + tree_head_clause = _get_hardware_tree_heads_clause(id_only=False) + + # We need a subquery because if we filter by any hardware, it will get the + # last head that has that hardware, but not the real head of the trees + query = f""" + WITH + -- Selects the data of the latest checkout of all trees in the given period + tree_heads AS ( + {tree_head_clause} + ) + SELECT DISTINCT + ON ( + TH.tree_name, + TH.git_repository_branch, + TH.git_repository_url, + TH.git_commit_hash + ) TH.tree_name, + TH.git_commit_hash + FROM + tests + INNER JOIN builds ON tests.build_id = builds.id + INNER JOIN tree_heads TH ON builds.checkout_id = TH.id + WHERE + ( + ( + tests.environment_compatible @> ARRAY[%(hardware)s]::TEXT[] + OR tests.environment_misc ->> 'platform' = %(hardware)s + ) + AND tests.origin = %(origin)s + AND TH.start_time >= %(start_date)s + AND TH.start_time <= %(end_date)s + ) + ORDER BY + TH.tree_name ASC, + TH.git_repository_branch ASC, + TH.git_repository_url ASC, + TH.git_commit_hash ASC, + TH.start_time DESC + """ + + params = { + "hardware": hardware_id, + "origin": origin, + "start_date": start_datetime, + "end_date": end_datetime, + } + trees = [] + with connection.cursor() as cursor: + cursor.execute(query, params) + tree_records = dict_fetchall(cursor) + trees = [ + (str(idx), tree["git_commit_hash"]) + for (idx, tree) in enumerate(tree_records) + ] + set_query_cache(key=cache_key, params=cache_params, rows=trees) + + return trees + + def get_hardware_trees_data( *, hardware_id: str, diff --git a/backend/kernelCI_app/tests/integrationTests/hardwareDetailsSummary_test.py b/backend/kernelCI_app/tests/integrationTests/hardwareDetailsSummary_test.py index 85c163d7b..a219d40f5 100644 --- a/backend/kernelCI_app/tests/integrationTests/hardwareDetailsSummary_test.py +++ b/backend/kernelCI_app/tests/integrationTests/hardwareDetailsSummary_test.py @@ -100,10 +100,11 @@ def pytest_generate_tests(metafunc): (ASUS_HARDWARE, {"boot.status": "MISS"}), (ASUS_HARDWARE, {"boot.status": "DONE"}), (ASUS_HARDWARE, {"boot.status": "NULL"}), + (ASUS_HARDWARE, {"boot.status": "null"}), (ASUS_HARDWARE, {"test.status": "ERROR"}), (ASUS_HARDWARE, {"test.status": "MISS"}), (ASUS_HARDWARE, {"test.status": "DONE"}), - (ASUS_HARDWARE, {"test.status": "NULL"}), + (ASUS_HARDWARE, {"test.status": "null"}), (ASUS_HARDWARE, {"boot.status": "PASS"}), (ASUS_HARDWARE, {"boot.status": "SKIP"}), (ASUS_HARDWARE, {"test.status": "SKIP"}), @@ -238,6 +239,7 @@ def test_filter_test_status(test_status_input): (ASUS_HARDWARE, {"build.status": "PASS"}), (ASUS_HARDWARE, {"build.status": "FAIL"}), (ASUS_HARDWARE, {"build.status": "NULL"}), + (ASUS_HARDWARE, {"build.status": "null"}), ], ) def test_filter_build_status(base_hardware, filters): @@ -266,7 +268,7 @@ def test_filter_build_status(base_hardware, filters): @pytest.mark.parametrize( "base_hardware, filters", [ - (ASUS_HARDWARE, {"config_name": "defconfig+kcidebug+x86-board"}), + (ASUS_HARDWARE, {"config_name": "defconfig"}), ], ) def test_filter_config_name(base_hardware, filters): @@ -289,7 +291,7 @@ def test_filter_config_name(base_hardware, filters): @pytest.mark.parametrize( "base_hardware, filters", [ - (ASUS_HARDWARE, {"architecture": "i386"}), + (ASUS_HARDWARE, {"architecture": "asus-CM1400CXA-dalboz"}), ], ) def test_filter_architectures(base_hardware, filters): @@ -420,7 +422,7 @@ def test_invalid_filters(invalid_filters_input): "fail_reasons": {}, "failed_platforms": [], "issues": [], - "platforms": None, + "platforms": {}, "status": { "ERROR": 0, "FAIL": 0, @@ -460,10 +462,13 @@ def test_invalid_filters(invalid_filters_input): filter, local_field = invalid_filters_input response, content = request_data(ASUS_HARDWARE, {filter: "invalid_filter,null"}) - assert_status_code(response=response, status_code=HTTPStatus.OK) - assert "error" not in content - assert "summary" in content - if local_field is None: - assert content["summary"] == empty_summary - else: - assert content["summary"][local_field] == empty_summary[local_field] + if "status" in filter: # enum literal values + assert_status_code(response=response, status_code=HTTPStatus.BAD_REQUEST) + else: # dynamic values + assert_status_code(response=response, status_code=HTTPStatus.OK) + assert "error" not in content + assert "summary" in content + if local_field is None: + assert content["summary"] == empty_summary + else: + assert content["summary"][local_field] == empty_summary[local_field] diff --git a/backend/kernelCI_app/typeModels/commonDetails.py b/backend/kernelCI_app/typeModels/commonDetails.py index 9fc804b3f..8daf16073 100644 --- a/backend/kernelCI_app/typeModels/commonDetails.py +++ b/backend/kernelCI_app/typeModels/commonDetails.py @@ -39,6 +39,28 @@ class TestArchSummaryItem(BaseModel): class BuildArchitectures(StatusCount): compilers: Optional[list[str]] = [] + def __iadd__(self, other: StatusCount) -> "BuildArchitectures": + self.PASS += other.PASS + self.ERROR += other.ERROR + self.FAIL += other.FAIL + self.SKIP += other.SKIP + self.MISS += other.MISS + self.DONE += other.DONE + self.NULL += other.NULL + return self + + def __add__(self, other: StatusCount) -> "BuildArchitectures": + return BuildArchitectures( + PASS=self.PASS + other.PASS, + ERROR=self.ERROR + other.ERROR, + FAIL=self.FAIL + other.FAIL, + SKIP=self.SKIP + other.SKIP, + MISS=self.MISS + other.MISS, + DONE=self.DONE + other.DONE, + NULL=self.NULL + other.NULL, + compilers=list(self.compilers or []), + ) + class TestHistoryItem(BaseModel): id: str diff --git a/backend/kernelCI_app/views/hardwareDetailsSummaryView.py b/backend/kernelCI_app/views/hardwareDetailsSummaryView.py index a5489df7d..bb7de3908 100644 --- a/backend/kernelCI_app/views/hardwareDetailsSummaryView.py +++ b/backend/kernelCI_app/views/hardwareDetailsSummaryView.py @@ -1,44 +1,38 @@ from collections import defaultdict from datetime import datetime +from itertools import chain +from typing import Dict, Optional from django.utils.decorators import method_decorator from django.views.decorators.csrf import csrf_exempt from drf_spectacular.utils import extend_schema from http import HTTPStatus import json -from kernelCI_app.helpers.commonDetails import PossibleTabs +from kernelCI_app.constants.general import UNKNOWN_STRING +from kernelCI_app.helpers.issueExtras import parse_issue from kernelCI_app.helpers.errorHandling import create_api_error_response +from kernelCI_app.helpers.filters import ( + FilterParams, + is_filtered_out, + is_valid_status, +) from kernelCI_app.helpers.hardwareDetails import ( - assign_default_record_values, - decide_if_is_build_in_filter, - decide_if_is_full_record_filtered_out, - decide_if_is_test_in_filter, generate_build_summary_typed, generate_test_summary_typed, - generate_tree_status_summary_dict, - get_build_typed, - get_filter_options, - get_processed_issue_key, - get_trees_with_selected_commit, - get_validated_current_tree, - handle_build_summary, - handle_test_summary, - handle_tree_status_summary, - is_issue_processed, - is_test_processed, - process_issue, - set_trees_status_summary, - format_issue_summary_for_response, unstable_parse_post_body, ) from kernelCI_app.queries.hardware import ( - get_hardware_details_data, - get_hardware_trees_data, + get_hardware_details_summary, + get_hardware_trees_head_commits, ) +from kernelCI_app.typeModels.common import StatusCount from kernelCI_app.typeModels.commonDetails import ( + BuildArchitectures, + BuildSummary, GlobalFilters, LocalFilters, Summary, TestArchSummaryItem, + TestSummary, ) from kernelCI_app.typeModels.commonOpenApiParameters import ( HARDWARE_ID_PATH_PARAM, @@ -49,14 +43,11 @@ HardwareDetailsPostBody, HardwareDetailsSummaryResponse, HardwareTestLocalFilters, - PossibleTestType, Tree, ) -from kernelCI_app.utils import is_boot from pydantic import ValidationError from rest_framework.response import Response from rest_framework.views import APIView -from typing import Dict, List, Set from kernelCI_app.constants.localization import ClientStrings @@ -67,199 +58,434 @@ # supported in this project @method_decorator(csrf_exempt, name="dispatch") class HardwareDetailsSummary(APIView): - def __init__(self): + + def __init__(self, **kwargs): + super().__init__(**kwargs) self.origin: str = None self.start_datetime: datetime = None self.end_datetime: datetime = None self.selected_commits: Dict[str, str] = None - - self.processed_builds = set() - self.processed_tests = set() - - self.processed_issues: Dict[str, Set[str]] = { - "build": set(), - "boot": set(), - "test": set(), + self.filters: FilterParams = None + + def get_filter_type( + self, is_build: bool, is_boot: bool, is_test: bool, **kwargs + ) -> str: + if is_build: + return "build" + if is_boot: + return "boot" + if is_test: + return "test" + raise ValueError("Invalid filter type") + + def get_summary_type( + self, is_build: bool, is_boot: bool, is_test: bool, **kwargs + ) -> str: + if is_build: + return "builds" + if is_boot: + return "boots" + if is_test: + return "tests" + raise ValueError("Invalid summary type") + + def filter_instance( + self, + *, + hardware_id: str, + config: str, + lab: str, + compiler: str, + architecture: str, + compatibles: set[str], + status: str, + known_issues: set[str], + is_build: bool, + is_boot: bool, + is_test: bool, + ) -> bool: + + filters: FilterParams = self.filters + filter_type = self.get_filter_type(is_build, is_boot, is_test) + status_filter_map = { + "build": filters.filterBuildStatus, + "boot": filters.filterBootStatus, + "test": filters.filterTestStatus, } + if is_filtered_out(status, status_filter_map[filter_type]): + return True + if is_filtered_out(compiler, filters.filterCompiler): + return True + if is_filtered_out(config, filters.filterConfigs): + return True + if is_filtered_out(lab, filters.filter_labs): + return True + if is_filtered_out(architecture, filters.filterArchitecture): + return True + + if filters.filterHardware and filters.filterHardware.isdisjoint(compatibles): + return True + + filtered_issues = filters.filterIssues.get(filter_type, set()) + if filtered_issues and not known_issues.issubset(filtered_issues): + return True + + return False + + def aggregate_summaries( + self, summary: list[dict], hardware_id: str + ) -> tuple[BuildSummary, TestSummary, TestSummary]: + builds_summary = generate_build_summary_typed() + tests_summary = generate_test_summary_typed() + boots_summary = generate_test_summary_typed() + + tests_summary.platforms = {} + boots_summary.platforms = {} + + # aggregation + for instance in summary: + status = instance["status"] + count = instance["count"] + incidents = instance["incidents_count"] + known_issues = set(parse_issue(issue) for issue in instance["known_issues"]) + compatibles = set(instance["environment_compatible"] or []) + config = instance["config_name"] or UNKNOWN_STRING + origin = instance["origin"] or UNKNOWN_STRING + lab = instance["lab"] or UNKNOWN_STRING + platform = instance["platform"] or UNKNOWN_STRING + is_build = instance["is_build"] + is_test = instance["is_test"] + is_boot = instance["is_boot"] + (compiler, architecture) = [ + (val or UNKNOWN_STRING).strip(" []''") + for val in (instance["compiler_arch"] or [None, None]) + ] + + status_count = StatusCount() + status_count.increment(status, count) + + if self.filter_instance( + hardware_id=hardware_id, + config=config, + lab=lab, + compiler=compiler, + architecture=architecture, + compatibles=compatibles, + status=status, + known_issues=known_issues, + is_build=is_build, + is_boot=is_boot, + is_test=is_test, + ): + continue - self.processed_compatibles: Set[str] = set() - - self.issue_dicts = { - "build": { - "issues": {}, - "failedWithUnknownIssues": 0, - }, - "boot": { - "issues": {}, - "failedWithUnknownIssues": 0, - }, - "test": { - "issues": {}, - "failedWithUnknownIssues": 0, - }, + if is_build: + self.increment_build( + builds_summary=builds_summary, + status_count=status_count, + architecture=architecture, + config=config, + lab=lab, + origin=origin, + incidents=incidents, + compiler=compiler, + ) + + elif is_boot: + self.increment_test( + tests_summary=boots_summary, + status_count=status_count, + config=config, + lab=lab, + origin=origin, + incidents=incidents, + architecture=architecture, + compiler=compiler, + platform=platform, + ) + + elif is_test: + self.increment_test( + tests_summary=tests_summary, + status_count=status_count, + config=config, + lab=lab, + origin=origin, + incidents=incidents, + architecture=architecture, + compiler=compiler, + platform=platform, + ) + + # ensure uniqueness on architecture and compilers (maybe we could change data structures???) + for summary in builds_summary.architectures.values(): + summary.compilers = sorted(set(summary.compilers or [])) + tests_summary_archs = defaultdict(StatusCount) + for item in tests_summary.architectures: + tests_summary_archs[(item.arch, item.compiler)] += item.status + tests_summary.architectures = [ + TestArchSummaryItem(arch=arch, compiler=compiler, status=status) + for (arch, compiler), status in tests_summary_archs.items() + ] + + boots_summary_archs = defaultdict(StatusCount) + for item in boots_summary.architectures: + boots_summary_archs[(item.arch, item.compiler)] += item.status + boots_summary.architectures = [ + TestArchSummaryItem(arch=arch, compiler=compiler, status=status) + for (arch, compiler), status in boots_summary_archs.items() + ] + + return (builds_summary, boots_summary, tests_summary) + + def increment_test( + self, + *, + tests_summary: TestSummary, + status_count: StatusCount, + config: str, + lab: str, + origin: str, + incidents: int, + architecture: str, + compiler: str, + platform: str, + ): + if config not in tests_summary.configs: + tests_summary.configs[config] = StatusCount() + if lab not in tests_summary.labs: + tests_summary.labs[lab] = StatusCount() + if origin not in tests_summary.origins: + tests_summary.origins[origin] = StatusCount() + if platform not in tests_summary.platforms: + tests_summary.platforms[platform] = StatusCount() + + tests_summary.status += status_count + tests_summary.configs[config] += status_count + tests_summary.labs[lab] += status_count + tests_summary.architectures.append( + TestArchSummaryItem( + arch=architecture, compiler=compiler, status=status_count + ) + ) + tests_summary.origins[origin] += status_count + tests_summary.platforms[platform] += status_count + if status_count.FAIL > 0: + tests_summary.unknown_issues += status_count.FAIL - incidents + + def increment_build( + self, + *, + builds_summary: BuildSummary, + status_count: StatusCount, + architecture: str, + config: str, + lab: str, + origin: str, + incidents: int, + compiler: str, + ): + if architecture not in builds_summary.architectures: + builds_summary.architectures[architecture] = BuildArchitectures( + compilers=[] + ) + if config not in builds_summary.configs: + builds_summary.configs[config] = StatusCount() + if lab not in builds_summary.labs: + builds_summary.labs[lab] = StatusCount() + if origin not in builds_summary.origins: + builds_summary.origins[origin] = StatusCount() + + builds_summary.status += status_count + builds_summary.configs[config] += status_count + builds_summary.labs[lab] += status_count + builds_summary.origins[origin] += status_count + builds_summary.architectures[architecture] += status_count + if compiler not in (builds_summary.architectures[architecture].compilers or []): + builds_summary.architectures[architecture].compilers.append(compiler) + if status_count.FAIL > 0: + builds_summary.unknown_issues += status_count.FAIL - incidents + + def aggregate_common( + self, summary: list[dict], hardware_id: str + ) -> tuple[list[Tree], list[str]]: + + all_trees: dict[tuple, Tree] = dict() + all_compatibles: set[str] = set() + + # aggregation + for instance in summary: + status = instance["status"] + count = instance["count"] + origin = instance["origin"] or UNKNOWN_STRING + compatibles = set(instance["environment_compatible"] or []) + tree_name = instance["tree_name"] + git_repository_url = instance["git_repository_url"] + git_repository_branch = instance["git_repository_branch"] + git_commit_name = instance["git_commit_name"] + git_commit_hash = instance["git_commit_hash"] + git_commit_tags = instance["git_commit_tags"] + + status_count = StatusCount() + status_count.increment(status, count) + + if not (tree_name, git_repository_url, git_repository_branch) in all_trees: + all_trees[(tree_name, git_repository_url, git_repository_branch)] = ( + Tree( + index="", # if we dont mind to sort, we can just use len(all_trees) + tree_name=tree_name, + git_repository_branch=git_repository_branch, + git_repository_url=git_repository_url, + head_git_commit_hash=git_commit_hash, + head_git_commit_name=git_commit_name, + head_git_commit_tag=git_commit_tags, + origin=origin, + selected_commit_status={ + "builds": StatusCount(), + "boots": StatusCount(), + "tests": StatusCount(), + }, + is_selected=None, + ) + ) + row_type = self.get_summary_type(**instance) + all_trees[ + (tree_name, git_repository_url, git_repository_branch) + ].selected_commit_status[row_type] += status_count + all_compatibles.update(compatibles or []) + + all_compatibles.discard(hardware_id) + + # not sure if it is worth sorting for index (but is also not slowing us down) + sorted_trees = sorted( + all_trees.values(), + key=lambda t: ( + t.tree_name or "", + t.git_repository_branch or "", + t.head_git_commit_name or "", + ), + ) + for i, tree in enumerate(sorted_trees): + tree.index = str(i) + + return sorted_trees, sorted(all_compatibles) + + def aggregate_filters( + self, + builds_summary: BuildSummary, + boots_summary: TestSummary, + tests_summary: TestSummary, + hardware_id: str, + ) -> tuple[ + GlobalFilters, LocalFilters, HardwareTestLocalFilters, HardwareTestLocalFilters + ]: + builds_configs = {*builds_summary.configs} + boots_configs = {*boots_summary.configs} + tests_configs = {*tests_summary.configs} + all_config = {*builds_configs, *boots_configs, *tests_configs} + + builds_architectures = {*builds_summary.architectures} + boots_architectures = {*[item.arch for item in boots_summary.architectures]} + tests_architectures = {*[item.arch for item in tests_summary.architectures]} + all_architectures = { + *builds_architectures, + *boots_architectures, + *tests_architectures, } - self.processed_architectures: Dict[str, Dict[str, TestArchSummaryItem]] = { - "build": {}, - "boot": {}, - "test": {}, + builds_compilers = { + *[ + compiler + for arch in builds_summary.architectures.values() + for compiler in (arch.compilers or []) + ] } - - self.unfiltered_build_issues = set() - self.unfiltered_boot_issues = set() - self.unfiltered_test_issues = set() - self.unfiltered_uncategorized_issue_flags: Dict[PossibleTabs, bool] = { - "build": False, - "boot": False, - "test": False, + boots_compilers = {*[item.compiler for item in boots_summary.architectures]} + tests_compilers = {*[item.compiler for item in tests_summary.architectures]} + all_compilers = { + *builds_compilers, + *boots_compilers, + *tests_compilers, } - self.unfiltered_boot_platforms = set() - self.unfiltered_test_platforms = set() - - self.global_configs = set() - self.global_architectures = set() - self.global_compilers = set() - - self.builds_summary = generate_build_summary_typed() - self.boots_summary = generate_test_summary_typed() - self.tests_summary = generate_test_summary_typed() - - self.tree_status_summary = defaultdict(generate_tree_status_summary_dict) - self.compatibles: List[str] = [] - - self.unfiltered_origins: dict[PossibleTabs, set[str]] = { - "build": set(), - "boot": set(), - "test": set(), + builds_issues_version = { + (item.id, item.version) for item in builds_summary.issues } - - self.unfiltered_labs: dict[PossibleTabs, set[str]] = { - "build": set(), - "boot": set(), - "test": set(), + boots_issues_version = { + (item.id, item.version) for item in boots_summary.issues + } + tests_issues_version = { + (item.id, item.version) for item in tests_summary.issues } - def _process_test(self, record: Dict) -> None: - is_record_boot = is_boot(record["path"]) - test_type_key: PossibleTestType = "boot" if is_record_boot else "test" - task_issue_summary = self.issue_dicts[test_type_key] - - is_test_processed_result = is_test_processed( - record=record, processed_tests=self.processed_tests - ) - is_issue_processed_result = is_issue_processed( - record=record, processed_issues=self.processed_issues[test_type_key] - ) - should_process_test = decide_if_is_test_in_filter( - instance=self, test_type=test_type_key, record=record + builds_labs = {*builds_summary.labs} + boots_labs = {*boots_summary.labs} + tests_labs = {*tests_summary.labs} + + builds_origins = {*builds_summary.origins} + boots_origins = {*boots_summary.origins} + tests_origins = {*tests_summary.origins} + + boots_platforms = {*(boots_summary.platforms or [])} + tests_platforms = {*(tests_summary.platforms or [])} + + return ( + GlobalFilters( + configs=[*all_config], + architectures=[*all_architectures], + compilers=[*all_compilers], + ), + LocalFilters( + issues=[*builds_issues_version], + origins=[*builds_origins], + has_unknown_issue=builds_summary.unknown_issues > 0, + labs=[*builds_labs], + ), + HardwareTestLocalFilters( + issues=[*boots_issues_version], + origins=[*boots_origins], + has_unknown_issue=boots_summary.unknown_issues > 0, + platforms=[*boots_platforms], + labs=[*boots_labs], + ), + HardwareTestLocalFilters( + issues=[*tests_issues_version], + origins=[*tests_origins], + has_unknown_issue=tests_summary.unknown_issues > 0, + platforms=[*tests_platforms], + labs=[*tests_labs], + ), ) - if ( - should_process_test - and not is_issue_processed_result - and is_test_processed_result - ): - process_issue( - record=record, - task_issues_dict=task_issue_summary, - issue_from="test", - ) - processed_issue_key = get_processed_issue_key(record=record) - self.processed_issues[test_type_key].add(processed_issue_key) - - if should_process_test and not is_test_processed_result: - task_summary = self.boots_summary if is_record_boot else self.tests_summary - handle_test_summary( - record=record, - task=task_summary, - issue_dict=task_issue_summary, - processed_archs=self.processed_architectures[test_type_key], + def valid_filter_status(self) -> bool: + filters: FilterParams = self.filters + return all( + is_valid_status(filter_status) + for filter_status in chain( + filters.filterBuildStatus, + filters.filterBootStatus, + filters.filterTestStatus, ) - self.processed_tests.add(record["id"]) - - def _process_build(self, record: Dict, tree_index: int) -> None: - build = get_build_typed(record, tree_index) - build_id = record["build_id"] - - should_process_build = decide_if_is_build_in_filter( - instance=self, - build=build, - processed_builds=self.processed_builds, - incident_test_id=record["incidents__test_id"], ) - if should_process_build: - handle_build_summary( - record=record, - builds_summary=self.builds_summary, - issue_dict=self.issue_dicts["build"], - tree_index=tree_index, - ) - self.processed_builds.add(build_id) - - def _sanitize_records( - self, records, trees: List[Tree], is_all_selected: bool - ) -> None: - for record in records: - current_tree = get_validated_current_tree( - record=record, selected_trees=trees - ) - if current_tree is None: - continue - - assign_default_record_values(record) - - if record["environment_compatible"] is not None: - self.processed_compatibles.update(record["environment_compatible"]) - - tree_index = current_tree.index - - handle_tree_status_summary( - record=record, - tree_status_summary=self.tree_status_summary, - tree_index=tree_index, - processed_builds=self.processed_builds, - ) - - is_record_filtered_out = decide_if_is_full_record_filtered_out( - instance=self, - record=record, - current_tree=current_tree, - is_all_selected=is_all_selected, - ) - if is_record_filtered_out: - continue - - self._process_test(record=record) - - self._process_build(record, tree_index) - - def _format_processing_for_response(self, hardware_id: str) -> None: - self.compatibles = list(self.processed_compatibles - {hardware_id}) - - self.boots_summary.architectures = list( - self.processed_architectures["boot"].values() - ) - self.tests_summary.architectures = list( - self.processed_architectures["test"].values() - ) - - format_issue_summary_for_response( - builds_summary=self.builds_summary, - boots_summary=self.boots_summary, - tests_summary=self.tests_summary, - issue_dicts=self.issue_dicts, - ) - - # Using post to receive a body request - @extend_schema( - parameters=[HARDWARE_ID_PATH_PARAM], - responses=HardwareDetailsSummaryResponse, - request=HardwareDetailsPostBody, - methods=["POST"], - ) - def post(self, request, hardware_id) -> Response: + def select_commits_hashes( + self, + tree_heads: list[(str, str)], + selected_commits: Optional[dict[str, str]] = None, + ): + selected_commit_hashes = [] + if selected_commits: + for idx, head in tree_heads: + if idx in self.selected_commits: + selected_commit = self.selected_commits.get(idx, "head") + selected_commit_hashes.append( + head if selected_commit == "head" else selected_commit + ) + else: + selected_commit_hashes = [head for (_, head) in tree_heads] + return selected_commit_hashes + + def _validate_request(self, request) -> Response | None: try: unstable_parse_post_body(instance=self, request=request) except ValidationError as e: @@ -278,102 +504,115 @@ def post(self, request, hardware_id) -> Response: status=HTTPStatus.BAD_REQUEST, ) - trees = get_hardware_trees_data( - hardware_id=hardware_id, - origin=self.origin, - start_datetime=self.start_datetime, - end_datetime=self.end_datetime, - ) - - if len(trees) == 0: + if not self.valid_filter_status(): return create_api_error_response( - error_message=ClientStrings.HARDWARE_NO_COMMITS, - status_code=HTTPStatus.OK, + error_message=ClientStrings.INVALID_FILTERS, + status_code=HTTPStatus.BAD_REQUEST, ) - - trees_with_selected_commits = get_trees_with_selected_commit( - trees=trees, selected_commits=self.selected_commits + return None + + def _get_error_response( + self, message: str, status_code: int = HTTPStatus.OK + ) -> Response: + return create_api_error_response( + error_message=message, + status_code=status_code, ) - records = get_hardware_details_data( - hardware_id=hardware_id, - origin=self.origin, - trees_with_selected_commits=trees_with_selected_commits, - start_datetime=self.start_datetime, - end_datetime=self.end_datetime, - ) + @extend_schema( + parameters=[HARDWARE_ID_PATH_PARAM], + responses=HardwareDetailsSummaryResponse, + request=HardwareDetailsPostBody, + methods=["POST"], + ) + def post(self, request, hardware_id) -> Response: + validation_error = self._validate_request(request) + if validation_error: + return validation_error - if len(records) == 0: - return create_api_error_response( - error_message=ClientStrings.HARDWARE_NOT_FOUND, - status_code=HTTPStatus.OK, + try: + + tree_heads = get_hardware_trees_head_commits( + hardware_id=hardware_id, + origin=self.origin, + start_datetime=self.start_datetime, + end_datetime=self.end_datetime, ) - is_all_selected = len(self.selected_commits) == 0 + if not tree_heads: + return self._get_error_response(ClientStrings.HARDWARE_NO_COMMITS) - try: - self._sanitize_records( - records, trees_with_selected_commits, is_all_selected + filters: FilterParams = self.filters + + selected_commit_hashes = self.select_commits_hashes( + tree_heads, self.selected_commits ) - self._format_processing_for_response(hardware_id=hardware_id) + summary: list[dict] = get_hardware_details_summary( + hardware_id=hardware_id, + origin=self.origin, + commit_hashes=selected_commit_hashes, + start_datetime=self.start_datetime, + end_datetime=self.end_datetime, + builds_duration=( + filters.filterBuildDurationMin, + filters.filterBuildDurationMax, + ), + boots_duration=( + filters.filterBootDurationMin, + filters.filterBootDurationMax, + ), + tests_duration=( + filters.filterTestDurationMin, + filters.filterTestDurationMax, + ), + ) - get_filter_options( - instance=self, - records=records, - selected_trees=trees_with_selected_commits, - is_all_selected=is_all_selected, + if not summary: + return self._get_error_response(ClientStrings.HARDWARE_NOT_FOUND) + + # TODO: necessary due to the fact we return filter info, + # a dedicated endpoint for filters is important + if filters.filters or self.selected_commits: + head_commit_hashes = self.select_commits_hashes(tree_heads) + unfiltered_summary = get_hardware_details_summary( + hardware_id=hardware_id, + origin=self.origin, + commit_hashes=head_commit_hashes, + start_datetime=self.start_datetime, + end_datetime=self.end_datetime, + ) + else: + unfiltered_summary = summary + + builds_summary, boots_summary, tests_summary = self.aggregate_summaries( + summary, hardware_id + ) + all_trees, all_compatibles = self.aggregate_common( + unfiltered_summary, hardware_id + ) + all_filters, builds_filters, boots_filters, tests_filters = ( + self.aggregate_filters( + *self.aggregate_summaries(unfiltered_summary, hardware_id), + hardware_id, + ) ) - set_trees_status_summary( - trees=trees, tree_status_summary=self.tree_status_summary + summary = Summary( + builds=builds_summary, boots=boots_summary, tests=tests_summary + ) + commons = HardwareCommon(trees=all_trees, compatibles=all_compatibles) + filters = HardwareDetailsFilters( + all=all_filters, + builds=builds_filters, + boots=boots_filters, + tests=tests_filters, ) valid_response = HardwareDetailsSummaryResponse( - summary=Summary( - builds=self.builds_summary, - boots=self.boots_summary, - tests=self.tests_summary, - ), - filters=HardwareDetailsFilters( - all=GlobalFilters( - configs=self.global_configs, - architectures=self.global_architectures, - compilers=self.global_compilers, - ), - builds=LocalFilters( - issues=list(self.unfiltered_build_issues), - has_unknown_issue=self.unfiltered_uncategorized_issue_flags[ - "build" - ], - origins=sorted(self.unfiltered_origins["build"]), - labs=sorted(self.unfiltered_labs["build"]), - ), - boots=HardwareTestLocalFilters( - issues=list(self.unfiltered_boot_issues), - platforms=list(self.unfiltered_boot_platforms), - has_unknown_issue=self.unfiltered_uncategorized_issue_flags[ - "boot" - ], - origins=sorted(self.unfiltered_origins["boot"]), - labs=sorted(self.unfiltered_labs["boot"]), - ), - tests=HardwareTestLocalFilters( - issues=list(self.unfiltered_test_issues), - platforms=list(self.unfiltered_test_platforms), - has_unknown_issue=self.unfiltered_uncategorized_issue_flags[ - "test" - ], - origins=sorted(self.unfiltered_origins["test"]), - labs=sorted(self.unfiltered_labs["test"]), - ), - ), - common=HardwareCommon( - trees=trees, - compatibles=self.compatibles, - ), + summary=summary, filters=filters, common=commons ) + + return Response(valid_response.model_dump()) except ValidationError as e: return Response(data=e.json(), status=HTTPStatus.INTERNAL_SERVER_ERROR) - - return Response(valid_response.model_dump()) diff --git a/dashboard/src/pages/hardwareDetails/HardwareDetails.tsx b/dashboard/src/pages/hardwareDetails/HardwareDetails.tsx index e44c2f10b..2c486038d 100644 --- a/dashboard/src/pages/hardwareDetails/HardwareDetails.tsx +++ b/dashboard/src/pages/hardwareDetails/HardwareDetails.tsx @@ -144,21 +144,6 @@ function HardwareDetails(): JSX.Element { ); }, [reqFilter, treeCommits, treeIndexes]); - const updateTreeFilters = useCallback((selectedIndexes: number[] | null) => { - navigate({ - search: previousSearch => ({ - ...previousSearch, - treeIndexes: selectedIndexes, - }), - state: s => s, - }); - // Since UpdateTreeFilters is used on a useEffect, anytime that updateTreeFilters is called - // it will trigger a rerender, changing the navigate object, changing this function, - // calling useEffect again on an infinite loop. - // TODO: check for uses of navigate as a dependency - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); - const onFilterChange = useCallback( (newFilter: TFilter) => { navigate({ @@ -203,6 +188,23 @@ function HardwareDetails(): JSX.Element { select: s => s.location.state.hardwareStatusCount, }); + const numIndexes = summaryResponse?.data?.common?.trees?.length || 0; + const updateTreeFilters = useCallback( + (selectedIndexes: number[] | null) => { + const numSelectedIndexes = selectedIndexes?.length || 0; + const indexes = + numSelectedIndexes === numIndexes ? null : selectedIndexes; + navigate({ + search: previousSearch => ({ + ...previousSearch, + treeIndexes: indexes, + }), + state: s => s, + }); + }, + [navigate, numIndexes], + ); + type HardwareStatusComparedState = typeof hardwareStatusHistoryState; const hardwareDataPreparedForInconsistencyValidation: HardwareStatusComparedState = diff --git a/dashboard/src/pages/hardwareDetails/HardwareDetailsHeaderTable.tsx b/dashboard/src/pages/hardwareDetails/HardwareDetailsHeaderTable.tsx index cdf51e740..51f098351 100644 --- a/dashboard/src/pages/hardwareDetails/HardwareDetailsHeaderTable.tsx +++ b/dashboard/src/pages/hardwareDetails/HardwareDetailsHeaderTable.tsx @@ -351,7 +351,6 @@ const getInitialRowSelection = ( const indexesFromRowSelection = ( rowSelection: RowSelectionState, - maxTreeItems: number, ): number[] | null => { const rowSelectionValues = Object.values(rowSelection); if (rowSelectionValues.length === 0) { @@ -362,10 +361,6 @@ const indexesFromRowSelection = ( parseInt(rowId), ); - if (selectedIndexes.length === maxTreeItems) { - return null; - } - return selectedIndexes; }; @@ -390,10 +385,7 @@ export function HardwareHeader({ const rowSelectionDebounced = useDebounce(rowSelection, DEBOUNCE_INTERVAL); useEffect(() => { - const updatedSelection = indexesFromRowSelection( - rowSelectionDebounced, - treeItems.length, - ); + const updatedSelection = indexesFromRowSelection(rowSelectionDebounced); updateTreeFilters(updatedSelection); }, [rowSelectionDebounced, updateTreeFilters, treeItems.length]); diff --git a/dashboard/src/types/hardware/hardwareDetails.ts b/dashboard/src/types/hardware/hardwareDetails.ts index a490cdde1..1c9ff4290 100644 --- a/dashboard/src/types/hardware/hardwareDetails.ts +++ b/dashboard/src/types/hardware/hardwareDetails.ts @@ -32,7 +32,7 @@ export type PreparedTrees = HardwareTrees & { isMainPageLoading: boolean; }; -type HardwareCommon = { +export type HardwareCommon = { trees: HardwareTrees[]; compatibles: string[]; };