diff --git a/backend/kernelCI_app/helpers/issueExtras.py b/backend/kernelCI_app/helpers/issueExtras.py index 74096ceb1..82647bc61 100644 --- a/backend/kernelCI_app/helpers/issueExtras.py +++ b/backend/kernelCI_app/helpers/issueExtras.py @@ -3,10 +3,14 @@ from kernelCI_app.constants.general import UNCATEGORIZED_STRING from kernelCI_app.helpers.logger import log_message -from kernelCI_app.queries.issues import get_issue_first_seen_data, get_issue_trees_data +from kernelCI_app.queries.issues import ( + get_issue_first_seen_data, + get_issue_last_seen_data, + get_issue_trees_data, +) from kernelCI_app.typeModels.issues import ( ExtraIssuesData, - FirstIncident, + Incident, IssueWithExtraInfo, ProcessedExtraDetailedIssues, TreeSetItem, @@ -29,6 +33,19 @@ def parse_issue(issue_str: Optional[str]) -> tuple[str, Optional[int]]: return (issue_id, issue_version) +def _incident_from_record(record: dict) -> Incident: + return Incident( + first_seen=record["first_seen"], + git_commit_hash=record["git_commit_hash"], + git_repository_url=record["git_repository_url"], + git_repository_branch=record["git_repository_branch"], + git_commit_name=record["git_commit_name"], + tree_name=record["tree_name"], + issue_version=record["issue_version"], + checkout_id=record["checkout_id"], + ) + + def process_issues_extra_details( *, issue_key_list: List[Tuple[str, int]], @@ -38,7 +55,7 @@ def process_issues_extra_details( return # TODO: combine both queries into one - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table, ) @@ -48,47 +65,43 @@ def process_issues_extra_details( ) -def assign_issue_first_seen( +def assign_issue_incidents( *, issue_key_list: List[Tuple[str, int]], processed_issues_table: ProcessedExtraDetailedIssues, ) -> None: """ - Assigns the first seen data to the processed_issues_table by querying with the issue_key_list. + Assigns first and last seen data to the processed_issues_table + by querying with the issue_key_list. """ - issue_id_set: set[str] = set() + issue_id_set = {issue_id for issue_id, _ in issue_key_list} versions_per_issue: dict[str, set[int]] = defaultdict(set) for issue_id, issue_version in issue_key_list: - issue_id_set.add(issue_id) versions_per_issue[issue_id].add(issue_version) - incident_records = get_issue_first_seen_data(issue_id_list=list(issue_id_set)) + first_incident_records = get_issue_first_seen_data(issue_id_list=list(issue_id_set)) + last_incident_records = get_issue_last_seen_data(issue_id_list=list(issue_id_set)) + last_incident_by_id = { + record["issue_id"]: record for record in last_incident_records + } - for record in incident_records: - record_issue_id = record["issue_id"] - first_seen = record["first_seen"] + for record in first_incident_records: + issue_id = record["issue_id"] + last_record = last_incident_by_id.get(issue_id) processed_issue_from_id = processed_issues_table.setdefault( - record_issue_id, + issue_id, ExtraIssuesData( - first_incident=FirstIncident( - first_seen=first_seen, - git_commit_hash=record["git_commit_hash"], - git_repository_url=record["git_repository_url"], - git_repository_branch=record["git_repository_branch"], - git_commit_name=record["git_commit_name"], - tree_name=record["tree_name"], - issue_version=record["issue_version"], - checkout_id=record["checkout_id"], - ), + first_incident=_incident_from_record(record), + last_incident=_incident_from_record(last_record), versions={}, ), ) # Initialize the versions table with null because that version may or may not exist. # If an issue_version exists, the trees can be assigned with `assign_issue_trees` - for version in versions_per_issue[record_issue_id]: + for version in versions_per_issue[issue_id]: processed_issue_from_id.versions.setdefault(version, None) diff --git a/backend/kernelCI_app/management/commands/helpers/summary.py b/backend/kernelCI_app/management/commands/helpers/summary.py index d6d0fded5..4c51a1fee 100644 --- a/backend/kernelCI_app/management/commands/helpers/summary.py +++ b/backend/kernelCI_app/management/commands/helpers/summary.py @@ -5,7 +5,7 @@ from django.conf import settings from kernelCI_app.constants.general import DEFAULT_ORIGIN -from kernelCI_app.helpers.issueExtras import assign_issue_first_seen +from kernelCI_app.helpers.issueExtras import assign_issue_incidents from kernelCI_app.helpers.logger import log_message from kernelCI_app.queries.notifications import ( get_issues_summary_data, @@ -134,7 +134,7 @@ def get_build_issues_from_checkout( issues_id_and_version_set.add((issue_id, issue_version)) processed_issues_table: ProcessedExtraDetailedIssues = {} - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=list(issues_id_and_version_set), processed_issues_table=processed_issues_table, ) diff --git a/backend/kernelCI_app/queries/issues.py b/backend/kernelCI_app/queries/issues.py index e6d11eacf..06cbc8809 100644 --- a/backend/kernelCI_app/queries/issues.py +++ b/backend/kernelCI_app/queries/issues.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any, Optional +from typing import Any, Literal, Optional from django.db import connection, connections @@ -228,39 +228,36 @@ def get_test_issues(*, test_id: str) -> list[dict]: return rows -def get_issue_first_seen_data(*, issue_id_list: list[str]) -> list[dict]: - """ - Retrieves the incident and checkout data - of the first incident of a list of issues - through a list of `issue_id`s. +def get_issue_seen_data( + *, issue_id_list: list[str], mode: Literal["first", "last"] = "first" +) -> list[dict]: """ + Retrieves the incident and checkout data of either the first or last + incident of a list of issues through a list of `issue_id`s. + :param mode: Either 'first' to get oldest incidents or 'last' to get the newest ones. + """ if not issue_id_list: return [] - cache_key = "issue_first_seen" + order_direction = "ASC" if mode == "first" else "DESC" + cache_key = f"issue_{mode}_seen" params = {"issue_id_list": issue_id_list} records = get_query_cache(key=cache_key, params=params) if records is None: - if len(issue_id_list) == 1: - comparison = "= %s" - else: - placeholders = ", ".join(["%s"] * len(issue_id_list)) - comparison = f"IN ({placeholders})" - query = f""" - WITH first_incident AS ( - SELECT DISTINCT - ON (IC.issue_id) IC.id + WITH target_incident AS ( + SELECT DISTINCT ON (IC.issue_id) + IC.id FROM incidents IC WHERE - IC.issue_id {comparison} + IC.issue_id = ANY(%(issue_id_list)s) ORDER BY IC.issue_id, - IC.issue_version ASC, - IC._timestamp ASC + IC.issue_version {order_direction}, + IC._timestamp {order_direction} ) SELECT IC.id, @@ -281,11 +278,11 @@ def get_issue_first_seen_data(*, issue_id_list: list[str]) -> list[dict]: OR T.build_id = B.id ) LEFT JOIN checkouts C ON B.checkout_id = C.id - JOIN first_incident FI ON IC.id = FI.id + JOIN target_incident TI ON IC.id = TI.id """ with connection.cursor() as cursor: - cursor.execute(query, issue_id_list) + cursor.execute(query, params) records = dict_fetchall(cursor) set_query_cache(key=cache_key, params=params, rows=records) @@ -293,6 +290,24 @@ def get_issue_first_seen_data(*, issue_id_list: list[str]) -> list[dict]: return records +def get_issue_first_seen_data(*, issue_id_list: list[str]) -> list[dict]: + """ + Retrieves the incident and checkout data + of the first incident of a list of issues + through a list of `issue_id`s. + """ + return get_issue_seen_data(issue_id_list=issue_id_list, mode="first") + + +def get_issue_last_seen_data(*, issue_id_list: list[str]) -> list[dict]: + """ + Retrieves the incident and checkout data + of the last incident of a list of issues + through a list of `issue_id`s. + """ + return get_issue_seen_data(issue_id_list=issue_id_list, mode="last") + + def get_issue_trees_data( *, issue_key_list: list[tuple[str, int]] ) -> list[dict[str, Any]]: diff --git a/backend/kernelCI_app/tests/unitTests/helpers/issueExtras_test.py b/backend/kernelCI_app/tests/unitTests/helpers/issueExtras_test.py index cf6684d17..43b28b6e8 100644 --- a/backend/kernelCI_app/tests/unitTests/helpers/issueExtras_test.py +++ b/backend/kernelCI_app/tests/unitTests/helpers/issueExtras_test.py @@ -3,22 +3,22 @@ from kernelCI_app.helpers.issueExtras import ( TagUrls, - assign_issue_first_seen, + assign_issue_incidents, assign_issue_trees, process_issues_extra_details, ) from kernelCI_app.typeModels.issues import ( ExtraIssuesData, - FirstIncident, + Incident, IssueWithExtraInfo, ) class TestProcessIssuesExtraDetails: - @patch("kernelCI_app.helpers.issueExtras.assign_issue_first_seen") + @patch("kernelCI_app.helpers.issueExtras.assign_issue_incidents") @patch("kernelCI_app.helpers.issueExtras.assign_issue_trees") def test_process_issues_extra_details_with_issues( - self, mock_assign_trees, mock_assign_first_seen + self, mock_assign_trees, mock_assign_incidents ): """Test process_issues_extra_details with issues.""" issue_key_list = [("issue1", 1), ("issue2", 2)] @@ -28,17 +28,17 @@ def test_process_issues_extra_details_with_issues( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) - mock_assign_first_seen.assert_called_once_with( + mock_assign_incidents.assert_called_once_with( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) mock_assign_trees.assert_called_once_with( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) - @patch("kernelCI_app.helpers.issueExtras.assign_issue_first_seen") + @patch("kernelCI_app.helpers.issueExtras.assign_issue_incidents") @patch("kernelCI_app.helpers.issueExtras.assign_issue_trees") def test_process_issues_extra_details_empty_list( - self, mock_assign_trees, mock_assign_first_seen + self, mock_assign_trees, mock_assign_incidents ): """Test process_issues_extra_details with empty issue list.""" issue_key_list = [] @@ -48,15 +48,18 @@ def test_process_issues_extra_details_empty_list( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) - mock_assign_first_seen.assert_not_called() + mock_assign_incidents.assert_not_called() mock_assign_trees.assert_not_called() class TestAssignIssueFirstSeen: + @patch("kernelCI_app.helpers.issueExtras.get_issue_last_seen_data") @patch("kernelCI_app.helpers.issueExtras.get_issue_first_seen_data") - def test_assign_issue_first_seen_with_data(self, mock_get_data): - """Test assign_issue_first_seen with data.""" - mock_get_data.return_value = [ + def test_assign_issue_first_seen_with_data( + self, mock_get_first_data, mock_get_last_data + ): + """Test assign_issue_incidents with data.""" + mock_get_first_data.return_value = [ { "issue_id": "issue1", "first_seen": "2024-01-15T10:00:00Z", @@ -69,15 +72,29 @@ def test_assign_issue_first_seen_with_data(self, mock_get_data): "checkout_id": "checkout1", } ] + mock_get_last_data.return_value = [ + { + "issue_id": "issue1", + "first_seen": "2024-06-15T10:00:00Z", + "git_commit_hash": "xyz789", + "git_repository_url": TagUrls.MAINLINE_URL, + "git_repository_branch": "master", + "git_commit_name": "commit_last", + "tree_name": "mainline", + "issue_version": 2, + "checkout_id": "checkout2", + } + ] issue_key_list = [("issue1", 1)] processed_issues_table = {} - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) - mock_get_data.assert_called_once_with(issue_id_list=["issue1"]) + mock_get_first_data.assert_called_once_with(issue_id_list=["issue1"]) + mock_get_last_data.assert_called_once_with(issue_id_list=["issue1"]) assert "issue1" in processed_issues_table issue_data = processed_issues_table["issue1"] @@ -86,12 +103,16 @@ def test_assign_issue_first_seen_with_data(self, mock_get_data): "2024-01-15T10:00:00Z" ) assert issue_data.first_incident.git_commit_hash == "abc123" + assert issue_data.last_incident.git_commit_hash == "xyz789" assert 1 in issue_data.versions + @patch("kernelCI_app.helpers.issueExtras.get_issue_last_seen_data") @patch("kernelCI_app.helpers.issueExtras.get_issue_first_seen_data") - def test_assign_issue_first_seen_with_multiple_versions(self, mock_get_data): - """Test assign_issue_first_seen with multiple versions.""" - mock_get_data.return_value = [ + def test_assign_issue_incidents_with_multiple_versions( + self, mock_get_first_data, mock_get_last_data + ): + """Test assign_issue_incidents with multiple versions.""" + mock_get_first_data.return_value = [ { "issue_id": "issue1", "first_seen": "2024-01-15T10:00:00Z", @@ -104,11 +125,12 @@ def test_assign_issue_first_seen_with_multiple_versions(self, mock_get_data): "checkout_id": "checkout1", } ] + mock_get_last_data.return_value = mock_get_first_data.return_value issue_key_list = [("issue1", 1), ("issue1", 2)] processed_issues_table = {} - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) @@ -124,10 +146,13 @@ def test_assign_issue_first_seen_with_multiple_versions(self, mock_get_data): assert issue_data.versions[1] is None assert issue_data.versions[2] is None + @patch("kernelCI_app.helpers.issueExtras.get_issue_last_seen_data") @patch("kernelCI_app.helpers.issueExtras.get_issue_first_seen_data") - def test_assign_issue_first_seen_with_multiple_issues(self, mock_get_data): - """Test assign_issue_first_seen with multiple issues.""" - mock_get_data.return_value = [ + def test_assign_issue_incidents_with_multiple_issues( + self, mock_get_first_data, mock_get_last_data + ): + """Test assign_issue_incidents with multiple issues.""" + mock_get_first_data.return_value = [ { "issue_id": "issue1", "first_seen": "2024-01-15T10:00:00Z", @@ -151,11 +176,12 @@ def test_assign_issue_first_seen_with_multiple_issues(self, mock_get_data): "checkout_id": "checkout2", }, ] + mock_get_last_data.return_value = mock_get_first_data.return_value issue_key_list = [("issue1", 1), ("issue2", 1)] processed_issues_table = {} - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) @@ -163,15 +189,19 @@ def test_assign_issue_first_seen_with_multiple_issues(self, mock_get_data): assert "issue2" in processed_issues_table assert len(processed_issues_table) == 2 + @patch("kernelCI_app.helpers.issueExtras.get_issue_last_seen_data") @patch("kernelCI_app.helpers.issueExtras.get_issue_first_seen_data") - def test_assign_issue_first_seen_no_data(self, mock_get_data): - """Test assign_issue_first_seen with no data.""" - mock_get_data.return_value = [] + def test_assign_issue_incidents_no_data( + self, mock_get_first_data, mock_get_last_data + ): + """Test assign_issue_incidents with no data.""" + mock_get_first_data.return_value = [] + mock_get_last_data.return_value = [] issue_key_list = [("issue1", 1)] processed_issues_table = {} - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) @@ -195,7 +225,17 @@ def test_assign_issue_trees_with_data(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), + last_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.MAINLINE_URL, @@ -239,7 +279,7 @@ def test_assign_issue_trees_with_stable_tag(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.STABLE_URL, @@ -249,6 +289,16 @@ def test_assign_issue_trees_with_stable_tag(self, mock_get_data): issue_version=1, checkout_id="checkout1", ), + last_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), versions={1: None}, ) } @@ -277,7 +327,7 @@ def test_assign_issue_trees_with_linux_next_tag(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.LINUX_NEXT_URL, @@ -287,6 +337,16 @@ def test_assign_issue_trees_with_linux_next_tag(self, mock_get_data): issue_version=1, checkout_id="checkout1", ), + last_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), versions={1: None}, ) } @@ -315,7 +375,7 @@ def test_assign_issue_trees_with_pending_fixes_branch(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.LINUX_NEXT_URL, @@ -325,6 +385,16 @@ def test_assign_issue_trees_with_pending_fixes_branch(self, mock_get_data): issue_version=1, checkout_id="checkout1", ), + last_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), versions={1: None}, ) } @@ -353,7 +423,17 @@ def test_assign_issue_trees_with_none_tree_name(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), + last_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.MAINLINE_URL, @@ -392,7 +472,17 @@ def test_assign_issue_trees_with_none_branch(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), + last_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.MAINLINE_URL, @@ -459,7 +549,17 @@ def test_assign_issue_trees_with_missing_version(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), + last_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.MAINLINE_URL, diff --git a/backend/kernelCI_app/tests/unitTests/queries/issues_queries_test.py b/backend/kernelCI_app/tests/unitTests/queries/issues_queries_test.py index 42cce3305..41ef1c0da 100644 --- a/backend/kernelCI_app/tests/unitTests/queries/issues_queries_test.py +++ b/backend/kernelCI_app/tests/unitTests/queries/issues_queries_test.py @@ -6,6 +6,7 @@ get_issue_builds, get_issue_details, get_issue_first_seen_data, + get_issue_last_seen_data, get_issue_listing_data, get_issue_tests, get_issue_trees_data, @@ -231,6 +232,68 @@ def test_get_issue_first_seen_data_empty_list(self, mock_get_cache): mock_get_cache.assert_not_called() +class TestGetIssueLastSeenData: + @patch("kernelCI_app.queries.issues.get_query_cache") + def test_get_issue_last_seen_data_from_cache(self, mock_get_cache): + issue_id_list = ["issue_1", "issue_2"] + cached_data = [{"id": "incident_1", "issue_id": "issue_1"}] + mock_get_cache.return_value = cached_data + + result = get_issue_last_seen_data(issue_id_list=issue_id_list) + + assert result == cached_data + + @patch("kernelCI_app.queries.issues.get_query_cache") + @patch("kernelCI_app.queries.issues.set_query_cache") + @patch("kernelCI_app.queries.issues.dict_fetchall") + @patch("kernelCI_app.queries.issues.connection") + def test_get_issue_last_seen_data_from_database( + self, mock_connection, mock_dict_fetchall, mock_set_cache, mock_get_cache + ): + mock_get_cache.return_value = None + expected_result = [{"id": "incident", "issue_id": "issue"}] + mock_dict_fetchall.return_value = expected_result + setup_mock_cursor(mock_connection) + + result = get_issue_last_seen_data(issue_id_list=["issue"]) + + assert result == expected_result + mock_set_cache.assert_called_once() + + @patch("kernelCI_app.queries.issues.get_query_cache") + @patch("kernelCI_app.queries.issues.set_query_cache") + @patch("kernelCI_app.queries.issues.dict_fetchall") + @patch("kernelCI_app.queries.issues.connection") + def test_get_issue_last_seen_data_multiple_issues( + self, mock_connection, mock_dict_fetchall, mock_set_cache, mock_get_cache + ): + mock_get_cache.return_value = None + expected_result = [ + {"id": "incident_1", "issue_id": "issue_1"}, + {"id": "incident_2", "issue_id": "issue_2"}, + ] + mock_dict_fetchall.return_value = expected_result + mock_cursor = setup_mock_cursor(mock_connection) + + result = get_issue_last_seen_data( + issue_id_list=["issue_1", "issue_2", "issue_3"] + ) + + assert result == expected_result + execute_call = mock_cursor.execute.call_args + query = execute_call[0][0] + assert "ANY(%(issue_id_list)s)" in query + assert "DESC" in query + mock_set_cache.assert_called_once() + + @patch("kernelCI_app.queries.issues.get_query_cache") + def test_get_issue_last_seen_data_empty_list(self, mock_get_cache): + result = get_issue_last_seen_data(issue_id_list=[]) + + assert result == [] + mock_get_cache.assert_not_called() + + class TestGetIssueTreesData: @patch("kernelCI_app.queries.issues.dict_fetchall") @patch("kernelCI_app.queries.issues.connections") diff --git a/backend/kernelCI_app/typeModels/issueListing.py b/backend/kernelCI_app/typeModels/issueListing.py index c1c0018ba..6b5d538eb 100644 --- a/backend/kernelCI_app/typeModels/issueListing.py +++ b/backend/kernelCI_app/typeModels/issueListing.py @@ -15,7 +15,7 @@ Origin, Timestamp, ) -from kernelCI_app.typeModels.issues import FirstIncident +from kernelCI_app.typeModels.issues import Incident class IssueListingQueryParameters(ListingInterval): @@ -53,5 +53,5 @@ class IssueListingFilters(BaseModel): class IssueListingResponse(BaseModel): issues: list[IssueListingItem] - extras: dict[str, FirstIncident] + extras: dict[str, Incident] filters: IssueListingFilters diff --git a/backend/kernelCI_app/typeModels/issues.py b/backend/kernelCI_app/typeModels/issues.py index 06ddbf489..878150cf9 100644 --- a/backend/kernelCI_app/typeModels/issues.py +++ b/backend/kernelCI_app/typeModels/issues.py @@ -67,7 +67,7 @@ class IssueExtraDetailsRequest(BaseModel): ) -class FirstIncident(BaseModel): +class Incident(BaseModel): first_seen: Timestamp git_commit_hash: Optional[Checkout__GitCommitHash] git_repository_url: Optional[Checkout__GitRepositoryUrl] @@ -79,7 +79,8 @@ class FirstIncident(BaseModel): class ExtraIssuesData(BaseModel): - first_incident: FirstIncident + first_incident: Incident + last_incident: Incident versions: dict[int, Optional[IssueWithExtraInfo]] diff --git a/backend/kernelCI_app/views/issueView.py b/backend/kernelCI_app/views/issueView.py index 0ed6f8e55..e092344ae 100644 --- a/backend/kernelCI_app/views/issueView.py +++ b/backend/kernelCI_app/views/issueView.py @@ -26,7 +26,7 @@ CULPRIT_CODE, CULPRIT_HARNESS, CULPRIT_TOOL, - FirstIncident, + Incident, PossibleIssueCulprits, ProcessedExtraDetailedIssues, ) @@ -51,7 +51,7 @@ def _resolve_issue_date_wrapper( class IssueView(APIView): def __init__(self): self.processed_extra_issue_details: ProcessedExtraDetailedIssues = {} - self.first_incidents: dict[str, FirstIncident] = {} + self.first_incidents: dict[str, Incident] = {} self.unprocessed_origins: set[str] = set() self.unprocessed_culprits: set[PossibleIssueCulprits] = set() diff --git a/backend/schema.yml b/backend/schema.yml index a28abf497..fe12505b9 100644 --- a/backend/schema.yml +++ b/backend/schema.yml @@ -648,7 +648,8 @@ paths: minimum: 0 title: End Days Ago type: integer - description: Number of days ago that marks the end of the metrics interval + description: Exclusive UTC day offset for the end of a [start, end) metrics + interval - in: query name: start_days_ago schema: @@ -656,7 +657,8 @@ paths: minimum: 0 title: Start Days Ago type: integer - description: Number of days ago that marks the start of the metrics interval + description: Inclusive UTC day offset for the start of a [start, end) metrics + interval tags: - metrics security: @@ -2501,7 +2503,9 @@ components: ExtraIssuesData: properties: first_incident: - $ref: '#/components/schemas/FirstIncident' + $ref: '#/components/schemas/Incident' + last_incident: + $ref: '#/components/schemas/Incident' versions: additionalProperties: anyOf: @@ -2511,53 +2515,10 @@ components: type: object required: - first_incident + - last_incident - versions title: ExtraIssuesData type: object - FirstIncident: - properties: - first_seen: - $ref: '#/components/schemas/Timestamp' - git_commit_hash: - anyOf: - - $ref: '#/components/schemas/Checkout__GitCommitHash' - - type: 'null' - git_repository_url: - anyOf: - - $ref: '#/components/schemas/Checkout__GitRepositoryUrl' - - type: 'null' - git_repository_branch: - anyOf: - - $ref: '#/components/schemas/Checkout__GitRepositoryBranch' - - type: 'null' - git_commit_name: - anyOf: - - $ref: '#/components/schemas/Checkout__GitCommitName' - - type: 'null' - tree_name: - anyOf: - - $ref: '#/components/schemas/Checkout__TreeName' - - type: 'null' - issue_version: - anyOf: - - $ref: '#/components/schemas/Issue__Version' - - type: 'null' - checkout_id: - anyOf: - - type: string - - type: 'null' - title: Checkout Id - required: - - first_seen - - git_commit_hash - - git_repository_url - - git_repository_branch - - git_commit_name - - tree_name - - issue_version - - checkout_id - title: FirstIncident - type: object GlobalFilters: properties: configs: @@ -2839,6 +2800,7 @@ components: - items: type: string type: array + uniqueItems: true - type: 'null' title: Hardware platform: @@ -3102,6 +3064,50 @@ components: - platforms title: HardwareTestLocalFilters type: object + Incident: + properties: + first_seen: + $ref: '#/components/schemas/Timestamp' + git_commit_hash: + anyOf: + - $ref: '#/components/schemas/Checkout__GitCommitHash' + - type: 'null' + git_repository_url: + anyOf: + - $ref: '#/components/schemas/Checkout__GitRepositoryUrl' + - type: 'null' + git_repository_branch: + anyOf: + - $ref: '#/components/schemas/Checkout__GitRepositoryBranch' + - type: 'null' + git_commit_name: + anyOf: + - $ref: '#/components/schemas/Checkout__GitCommitName' + - type: 'null' + tree_name: + anyOf: + - $ref: '#/components/schemas/Checkout__TreeName' + - type: 'null' + issue_version: + anyOf: + - $ref: '#/components/schemas/Issue__Version' + - type: 'null' + checkout_id: + anyOf: + - type: string + - type: 'null' + title: Checkout Id + required: + - first_seen + - git_commit_hash + - git_repository_url + - git_repository_branch + - git_commit_name + - tree_name + - issue_version + - checkout_id + title: Incident + type: object Issue: properties: id: @@ -3308,7 +3314,7 @@ components: type: array extras: additionalProperties: - $ref: '#/components/schemas/FirstIncident' + $ref: '#/components/schemas/Incident' title: Extras type: object filters: @@ -3578,6 +3584,13 @@ components: type: array title: Top Issues By Origin type: object + new_issues_by_origin: + additionalProperties: + items: + $ref: '#/components/schemas/TopIssue' + type: array + title: New Issues By Origin + type: object lab_maps: additionalProperties: $ref: '#/components/schemas/LabMetricsData' @@ -3609,6 +3622,7 @@ components: - n_incidents - build_incidents_by_origin - top_issues_by_origin + - new_issues_by_origin - lab_maps - prev_n_trees - prev_n_checkouts diff --git a/dashboard/src/components/IssueDetails/IssueDetails.tsx b/dashboard/src/components/IssueDetails/IssueDetails.tsx index 0c83bf10f..db085ed19 100644 --- a/dashboard/src/components/IssueDetails/IssueDetails.tsx +++ b/dashboard/src/components/IssueDetails/IssueDetails.tsx @@ -34,7 +34,7 @@ import { import { BranchBadge } from '@/components/Badge/BranchBadge'; import { getLogspecSection } from '@/components/Section/LogspecSection'; -import { getFirstIncidentSection } from '@/components/Section/FirstIncidentSection'; +import { getIncidentsSection } from '@/components/Section/FirstIncidentSection'; import PageWithTitle from '@/components/PageWithTitle'; @@ -102,9 +102,11 @@ export const IssueDetails = ({ }, [data?.misc, formatMessage]); const firstIncidentSection: ISection | undefined = useMemo(() => { - return getFirstIncidentSection({ + return getIncidentsSection({ firstIncident: data?.extra?.[issueId]?.first_incident, + lastIncident: data?.extra?.[issueId]?.last_incident, title: formatMessage({ id: 'issueDetails.firstIncidentData' }), + lastIncidentTitle: formatMessage({ id: 'issueDetails.lastIncident' }), }); }, [data?.extra, formatMessage, issueId]); diff --git a/dashboard/src/components/OpenGraphTags/IssueDetailsOGTags.tsx b/dashboard/src/components/OpenGraphTags/IssueDetailsOGTags.tsx index d5ba30dcc..24f6a5b50 100644 --- a/dashboard/src/components/OpenGraphTags/IssueDetailsOGTags.tsx +++ b/dashboard/src/components/OpenGraphTags/IssueDetailsOGTags.tsx @@ -36,10 +36,18 @@ const IssueDetailsOGTags = ({ new Date(firstSeen).toLocaleDateString() : ''; + const lastSeen = data.extra?.[issueId]?.last_incident?.first_seen; + const lastSeenDescription = lastSeen + ? formatMessage({ id: 'issue.lastSeen' }) + + ': ' + + new Date(lastSeen).toLocaleDateString() + : ''; + const descriptionChunks = [ versionDescription, culpritDescription, firstSeenDescription, + lastSeenDescription, ].filter(chunk => chunk !== ''); return descriptionChunks.join(';\n'); diff --git a/dashboard/src/components/Section/FirstIncidentSection.tsx b/dashboard/src/components/Section/FirstIncidentSection.tsx index 6c492fb04..bdfb73399 100644 --- a/dashboard/src/components/Section/FirstIncidentSection.tsx +++ b/dashboard/src/components/Section/FirstIncidentSection.tsx @@ -1,78 +1,91 @@ import { shouldTruncate, valueOrEmpty } from '@/lib/string'; -import type { FirstIncident } from '@/types/issueExtras'; +import type { Incident } from '@/types/issueExtras'; import { TooltipDateTime } from '@/components/TooltipDateTime'; import { TruncatedValueTooltip } from '@/components/Tooltip/TruncatedValueTooltip'; import { TreeDetailsLink } from '@/components/TreeDetailsLink/TreeDetailsLink'; -import type { ISection, SubsectionLink } from './Section'; +import type { ISection, ISubsection, SubsectionLink } from './Section'; -export const getFirstIncidentSection = ({ +const getIncidentInfos = (firstIncident: Incident): SubsectionLink[] => [ + { + title: 'global.treeBranchHash', + linkText: ( + + ), + }, + { + title: 'commonDetails.gitCommitName', + linkText: valueOrEmpty(firstIncident.git_commit_name), + }, + { + title: 'commonDetails.gitRepositoryUrl', + linkText: shouldTruncate(valueOrEmpty(firstIncident.git_repository_url)) ? ( + + ) : ( + valueOrEmpty(firstIncident.git_repository_url) + ), + link: firstIncident.git_repository_url, + }, + { + title: 'issue.seen', + linkText: ( + + ), + }, + { + title: 'issueDetails.firstIncidentVersion', + linkText: firstIncident.issue_version, + }, +]; + +export const getIncidentsSection = ({ firstIncident, + lastIncident, title, + lastIncidentTitle, }: { - firstIncident?: FirstIncident; + firstIncident?: Incident; + lastIncident?: Incident; title: string; + lastIncidentTitle?: string; }): ISection | undefined => { - if (!firstIncident) { + if (!firstIncident && !lastIncident) { return; } - const firstIncidentInfos: SubsectionLink[] = [ - { - title: 'global.treeBranchHash', - linkText: ( - - ), - }, - { - title: 'commonDetails.gitCommitName', - linkText: valueOrEmpty(firstIncident.git_commit_name), - }, - { - title: 'commonDetails.gitRepositoryUrl', - linkText: shouldTruncate( - valueOrEmpty(firstIncident.git_repository_url), - ) ? ( - - ) : ( - valueOrEmpty(firstIncident.git_repository_url) - ), - link: firstIncident.git_repository_url, - }, - { - title: 'issue.firstSeen', - linkText: ( - - ), - }, - { - title: 'issueDetails.firstIncidentVersion', - linkText: firstIncident.issue_version, - }, - ]; + const subsections: ISubsection[] = []; + + if (firstIncident) { + subsections.push({ + infos: getIncidentInfos(firstIncident), + }); + } + + if (lastIncident) { + subsections.push({ + title: lastIncidentTitle, + infos: getIncidentInfos(lastIncident), + }); + } return { title: title, - subsections: [ - { - infos: firstIncidentInfos, - }, - ], + subsections: subsections, }; }; diff --git a/dashboard/src/components/Section/Section.tsx b/dashboard/src/components/Section/Section.tsx index 800b2e928..01ce44914 100644 --- a/dashboard/src/components/Section/Section.tsx +++ b/dashboard/src/components/Section/Section.tsx @@ -93,13 +93,28 @@ export const Subsection = ({ infos, title }: ISubsection): JSX.Element => { return (
- {title && {title}} - {items.length > 0 && ( -
- {items} + {title ? ( +
+ + {title} + + {items.length > 0 && ( +
+ {items} +
+ )} + {children.length > 0 &&
{children}
}
+ ) : ( + <> + {items.length > 0 && ( +
+ {items} +
+ )} + {children.length > 0 &&
{children}
} + )} - {children.length > 0 &&
{children}
}
); }; diff --git a/dashboard/src/locales/messages/index.ts b/dashboard/src/locales/messages/index.ts index 45f5b13cc..b795e3419 100644 --- a/dashboard/src/locales/messages/index.ts +++ b/dashboard/src/locales/messages/index.ts @@ -242,10 +242,12 @@ export const messages = { 'hardwareListing.treeSelectorSearchPlaceholder': 'Search tree...', 'issue.alsoPresentTooltip': 'Issue also present in {tree}', 'issue.firstSeen': 'First seen', + 'issue.lastSeen': 'Last seen', 'issue.newIssue': 'New issue: This is the first time this issue was seen', 'issue.noIssueFound': 'No issues found for this tree/branch/commit', 'issue.path': 'Issues', 'issue.searchPlaceholder': 'Search by issue comment with a regex', + 'issue.seen': 'Seen', 'issue.tooltip': 'Issues groups several builds or tests by matching result status and logs.{br}They may also be linked to an external issue tracker or mailing list discussion.', 'issue.uncategorized': 'Uncategorized', @@ -260,6 +262,7 @@ export const messages = { 'issueDetails.issueDetails': 'Issue Details', 'issueDetails.issueListingInfo': 'The culprit for all issues listed is code', + 'issueDetails.lastIncident': 'Last Incident Data', 'issueDetails.logspecData': 'Logspec Data', 'issueDetails.noBuildResults': 'No builds associated with this issue', 'issueDetails.noTestResults': 'No tests associated with this issue', diff --git a/dashboard/src/types/issueExtras.ts b/dashboard/src/types/issueExtras.ts index 30ebffb3c..9ddb887ae 100644 --- a/dashboard/src/types/issueExtras.ts +++ b/dashboard/src/types/issueExtras.ts @@ -10,7 +10,7 @@ type TIssueVersionData = IssueKeys & { export type IssueKeyList = [string, number][]; -export type FirstIncident = { +export type Incident = { first_seen: Date; git_commit_hash?: string; git_repository_url?: string; @@ -22,7 +22,8 @@ export type FirstIncident = { }; type TExtraIssuesData = { - first_incident: FirstIncident; + first_incident: Incident; + last_incident: Incident; versions: Record; }; diff --git a/dashboard/src/types/issueListing.ts b/dashboard/src/types/issueListing.ts index 451342790..772565d9e 100644 --- a/dashboard/src/types/issueListing.ts +++ b/dashboard/src/types/issueListing.ts @@ -1,4 +1,4 @@ -import type { FirstIncident } from './issueExtras'; +import type { Incident } from './issueExtras'; import type { IssueKeys } from './issues'; export type IssueListingItem = IssueKeys & { @@ -19,8 +19,8 @@ export type IssueListingFilters = { export type IssueListingResponse = { issues: IssueListingItem[]; - extras: Record; + extras: Record; filters: IssueListingFilters; }; -export type IssueListingTableItem = IssueListingItem & FirstIncident; +export type IssueListingTableItem = IssueListingItem & Incident;