diff --git a/tests/perf/auto_perf_sheriffing/performance_alerting/__init__.py b/tests/perf/auto_perf_sheriffing/performance_alerting/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/perf/auto_perf_sheriffing/performance_alerting/conftest.py b/tests/perf/auto_perf_sheriffing/performance_alerting/conftest.py new file mode 100644 index 00000000000..9e497a4f85e --- /dev/null +++ b/tests/perf/auto_perf_sheriffing/performance_alerting/conftest.py @@ -0,0 +1,44 @@ +import datetime + +import pytest + +from treeherder.perf.models import PerformanceAlertSummary + + +@pytest.fixture +def test_perf_alert_summary_for_modifier( + test_repository, push_stored, test_perf_framework, test_issue_tracker +): + return PerformanceAlertSummary.objects.create( + repository=test_repository, + framework=test_perf_framework, + prev_push_id=1, + push_id=2, + manually_created=False, + created=datetime.datetime.now(), + bug_number=None, + ) + + +@pytest.fixture +def create_perf_alert_summary( + test_repository, test_perf_framework, push_stored, test_issue_tracker +): + counter = {"value": 3} + + def _create_summary(**kwargs): + push_id = counter["value"] + counter["value"] += 1 + defaults = { + "repository": test_repository, + "framework": test_perf_framework, + "prev_push_id": push_id, + "push_id": push_id + 1, + "manually_created": False, + "created": datetime.datetime.now(), + "bug_number": None, + } + defaults.update(kwargs) + return PerformanceAlertSummary.objects.create(**defaults) + + return _create_summary diff --git a/tests/perf/auto_perf_sheriffing/performance_alerting/test_alert_modifier.py b/tests/perf/auto_perf_sheriffing/performance_alerting/test_alert_modifier.py new file mode 100644 index 00000000000..b3942ae1454 --- /dev/null +++ b/tests/perf/auto_perf_sheriffing/performance_alerting/test_alert_modifier.py @@ -0,0 +1,208 @@ +from datetime import datetime +from unittest.mock import Mock, patch + +import pytest + +from treeherder.perf.auto_perf_sheriffing.performance_alerting.alert_modifier import ( + PerformanceAlertSummaryModifier, +) +from treeherder.perf.models import PerformanceAlertSummary + + +class TestResolutionModifier: + @pytest.fixture + def resolution_modifier_class(self): + for updater in PerformanceAlertSummaryModifier.get_updaters(): + if updater.__name__ == "ResolutionModifier": + return updater + pytest.fail("ResolutionModifier not found in updaters list") + + @patch("treeherder.perf.auto_perf_sheriffing.performance_alerting.alert_modifier.BugSearcher") + def test_update_alerts_no_bugs(self, mock_bug_searcher_class, resolution_modifier_class): + mock_searcher = Mock() + mock_searcher.get_today_date.return_value = datetime.now().date() + mock_searcher.get_bugs.return_value = {"bugs": []} + mock_bug_searcher_class.return_value = mock_searcher + + updates, summaries = resolution_modifier_class.update_alerts() + + assert updates == {} + assert summaries == {} + + @pytest.mark.parametrize( + "resolution, expected_bug_status", + [ + ("FIXED", PerformanceAlertSummary.BUG_FIXED), + ("INVALID", PerformanceAlertSummary.BUG_INVALID), + ("WONTFIX", PerformanceAlertSummary.BUG_WONTFIX), + ("DUPLICATE", PerformanceAlertSummary.BUG_DUPLICATE), + ("WORKSFORME", PerformanceAlertSummary.BUG_WORKSFORME), + ("INCOMPLETE", PerformanceAlertSummary.BUG_INCOMPLETE), + ("MOVED", PerformanceAlertSummary.BUG_MOVED), + ], + ) + @patch("treeherder.perf.auto_perf_sheriffing.performance_alerting.alert_modifier.BugSearcher") + def test_update_alerts_resolution_mapping( + self, + mock_bug_searcher_class, + resolution, + expected_bug_status, + resolution_modifier_class, + test_perf_alert_summary_for_modifier, + ): + mock_searcher = Mock() + mock_searcher.get_today_date.return_value = datetime.now().date() + mock_searcher.get_bugs.return_value = { + "bugs": [{"id": 12345, "resolution": resolution, "status": "RESOLVED"}] + } + mock_bug_searcher_class.return_value = mock_searcher + + test_perf_alert_summary_for_modifier.bug_number = 12345 + test_perf_alert_summary_for_modifier.save() + + updates, summaries = resolution_modifier_class.update_alerts() + + assert ( + updates[str(test_perf_alert_summary_for_modifier.id)]["bug_status"] + == expected_bug_status + ) + assert str(test_perf_alert_summary_for_modifier.id) in summaries + + @patch("treeherder.perf.auto_perf_sheriffing.performance_alerting.alert_modifier.BugSearcher") + def test_update_alerts_with_multiple_bugs( + self, + mock_bug_searcher_class, + resolution_modifier_class, + test_perf_alert_summary_for_modifier, + create_perf_alert_summary, + ): + mock_searcher = Mock() + mock_searcher.get_today_date.return_value = datetime.now().date() + mock_searcher.get_bugs.return_value = { + "bugs": [ + {"id": 12345, "resolution": "FIXED", "status": "RESOLVED"}, + {"id": 67890, "resolution": "INVALID", "status": "RESOLVED"}, + ] + } + mock_bug_searcher_class.return_value = mock_searcher + + summary2 = create_perf_alert_summary(bug_number=67890) + test_perf_alert_summary_for_modifier.bug_number = 12345 + test_perf_alert_summary_for_modifier.save() + + updates, summaries = resolution_modifier_class.update_alerts() + + assert ( + updates[str(test_perf_alert_summary_for_modifier.id)]["bug_status"] + == PerformanceAlertSummary.BUG_FIXED + ) + assert updates[str(summary2.id)]["bug_status"] == PerformanceAlertSummary.BUG_INVALID + assert str(test_perf_alert_summary_for_modifier.id) in summaries + assert str(summary2.id) in summaries + + @patch("treeherder.perf.auto_perf_sheriffing.performance_alerting.alert_modifier.BugSearcher") + def test_update_alerts_bug_not_matching_any_summary( + self, + mock_bug_searcher_class, + resolution_modifier_class, + test_perf_alert_summary_for_modifier, + ): + mock_searcher = Mock() + mock_searcher.get_today_date.return_value = datetime.now().date() + mock_searcher.get_bugs.return_value = { + "bugs": [{"id": 99999, "resolution": "FIXED", "status": "RESOLVED"}] + } + mock_bug_searcher_class.return_value = mock_searcher + + test_perf_alert_summary_for_modifier.bug_number = 12345 + test_perf_alert_summary_for_modifier.save() + + updates, summaries = resolution_modifier_class.update_alerts() + + assert updates == {} + assert summaries == {} + + @patch("treeherder.perf.auto_perf_sheriffing.performance_alerting.alert_modifier.BugSearcher") + def test_update_alerts_unknown_resolution_maps_to_bug_new( + self, + mock_bug_searcher_class, + resolution_modifier_class, + test_perf_alert_summary_for_modifier, + ): + mock_searcher = Mock() + mock_searcher.get_today_date.return_value = datetime.now().date() + mock_searcher.get_bugs.return_value = { + "bugs": [{"id": 12345, "resolution": "UNKNOWN_RESOLUTION", "status": "RESOLVED"}] + } + mock_bug_searcher_class.return_value = mock_searcher + + test_perf_alert_summary_for_modifier.bug_number = 12345 + test_perf_alert_summary_for_modifier.save() + + updates, summaries = resolution_modifier_class.update_alerts() + + assert updates == {} + assert summaries == {} + + @patch("treeherder.perf.auto_perf_sheriffing.performance_alerting.alert_modifier.BugSearcher") + def test_update_alerts_with_empty_resolution_defaults_to_bug_new( + self, + mock_bug_searcher_class, + resolution_modifier_class, + test_perf_alert_summary_for_modifier, + ): + mock_searcher = Mock() + mock_searcher.get_today_date.return_value = datetime.now().date() + mock_searcher.get_bugs.return_value = { + "bugs": [{"id": 12345, "resolution": "", "status": "ASSIGNED"}] + } + mock_bug_searcher_class.return_value = mock_searcher + + test_perf_alert_summary_for_modifier.bug_number = 12345 + test_perf_alert_summary_for_modifier.save() + + updates, summaries = resolution_modifier_class.update_alerts() + + assert ( + updates[str(test_perf_alert_summary_for_modifier.id)]["bug_status"] + == PerformanceAlertSummary.BUG_NEW + ) + assert str(test_perf_alert_summary_for_modifier.id) in summaries + + @patch("treeherder.perf.auto_perf_sheriffing.performance_alerting.alert_modifier.BugSearcher") + def test_update_alerts_skips_summary_already_at_target_status( + self, + mock_bug_searcher_class, + resolution_modifier_class, + test_perf_alert_summary_for_modifier, + ): + mock_searcher = Mock() + mock_searcher.get_today_date.return_value = datetime.now().date() + mock_searcher.get_bugs.return_value = { + "bugs": [{"id": 12345, "resolution": "FIXED", "status": "RESOLVED"}] + } + mock_bug_searcher_class.return_value = mock_searcher + + test_perf_alert_summary_for_modifier.bug_number = 12345 + test_perf_alert_summary_for_modifier.bug_status = PerformanceAlertSummary.BUG_FIXED + test_perf_alert_summary_for_modifier.save() + + updates, summaries = resolution_modifier_class.update_alerts() + + assert str(test_perf_alert_summary_for_modifier.id) not in updates + + @patch("treeherder.perf.auto_perf_sheriffing.performance_alerting.alert_modifier.BugSearcher") + def test_update_alerts_exception_handling( + self, mock_bug_searcher_class, resolution_modifier_class, caplog + ): + mock_searcher = Mock() + mock_searcher.get_today_date.return_value = datetime.now().date() + mock_searcher.get_bugs.side_effect = Exception("API Error") + mock_bug_searcher_class.return_value = mock_searcher + + updates, summaries = resolution_modifier_class.update_alerts() + + assert updates == {} + assert summaries == {} + assert "Failed to get bugs for alert resolution updates" in caplog.text + assert "API Error" in caplog.text diff --git a/treeherder/perf/auto_perf_sheriffing/performance_alerting/alert_manager.py b/treeherder/perf/auto_perf_sheriffing/performance_alerting/alert_manager.py new file mode 100644 index 00000000000..f616cbf5e3d --- /dev/null +++ b/treeherder/perf/auto_perf_sheriffing/performance_alerting/alert_manager.py @@ -0,0 +1,60 @@ +import logging + +from treeherder.perf.auto_perf_sheriffing.base_alert_manager import AlertManager +from treeherder.perf.auto_perf_sheriffing.performance_alerting.alert_modifier import ( + PerformanceAlertSummaryModifier, +) +from treeherder.perf.models import PerformanceAlertSummary + +MODIFIABLE_ALERT_SUMMARY_FIELDS = ("bug_status",) + +logger = logging.getLogger(__name__) + + +class PerformanceAlertManager(AlertManager): + def __init__(self): + super().__init__(bug_manager=None, email_manager=None) + + def update_alerts(self, alerts, *args, **kwargs): + alert_updates, summaries_with_updates = ( + PerformanceAlertSummaryModifier.get_alert_summary_updates() + ) + if not alert_updates: + return + + summaries_to_update = set() + fields_to_update = set() + + for summary_id, summary in summaries_with_updates.items(): + updates = alert_updates.get(summary_id) + if not updates: + continue + for field, value in updates.items(): + if field not in MODIFIABLE_ALERT_SUMMARY_FIELDS: + continue + summaries_to_update.add(summary) + fields_to_update.add(field) + logger.info( + f"Summary ID {summary_id}, {field}: {getattr(summary, field)} to {value}" + ) + setattr(summary, field, value) + + num_updated = PerformanceAlertSummary.objects.bulk_update( + list(summaries_to_update), list(fields_to_update) + ) + logger.info(f"{num_updated} summaries updated") + + def comment_alert_bugs(self, alerts, *args, **kwargs): + pass + + def file_alert_bugs(self, alerts, *args, **kwargs): + pass + + def modify_alert_bugs(self, alerts, *args, **kwargs): + pass + + def email_alerts(self, alerts, *args, **kwargs): + pass + + def house_keeping(self, alerts, *args, **kwargs): + pass diff --git a/treeherder/perf/auto_perf_sheriffing/performance_alerting/alert_modifier.py b/treeherder/perf/auto_perf_sheriffing/performance_alerting/alert_modifier.py new file mode 100644 index 00000000000..671334ed121 --- /dev/null +++ b/treeherder/perf/auto_perf_sheriffing/performance_alerting/alert_modifier.py @@ -0,0 +1,80 @@ +import logging +from datetime import timedelta + +from treeherder.perf.auto_perf_sheriffing.bug_searcher import BugSearcher +from treeherder.perf.models import PerformanceAlertSummary + +logger = logging.getLogger(__name__) + + +class PerformanceAlertSummaryModifier: + updaters = [] + + @staticmethod + def add(updater_class): + PerformanceAlertSummaryModifier.updaters.append(updater_class) + + @staticmethod + def get_updaters(): + return PerformanceAlertSummaryModifier.updaters + + @staticmethod + def get_alert_summary_updates(*args, **kwargs): + all_updates = {} + all_summaries_to_update = {} + for updater in PerformanceAlertSummaryModifier.get_updaters(): + updates, summaries_to_update = updater.update_alerts(**kwargs) + if not updates: + continue + all_updates.update(updates) + all_summaries_to_update.update(summaries_to_update) + return all_updates, all_summaries_to_update + + +@PerformanceAlertSummaryModifier.add +class ResolutionModifier: + @staticmethod + def update_alerts(**kwargs): + bug_searcher = BugSearcher() + start_date = bug_searcher.get_today_date() - timedelta(days=7) + bug_searcher.set_include_fields(["id", "resolution", "status"]) + bug_searcher.set_query( + { + "f1": "keywords", + "o1": "anywords", + "v1": "perf-alert", + "f2": "resolution", + "o2": "changedafter", + "v2": start_date, + } + ) + + try: + bugs = bug_searcher.get_bugs() + except Exception as e: + logger.warning(f"Failed to get bugs for alert resolution updates: {str(e)}") + return ({}, {}) + + alert_summaries = PerformanceAlertSummary.objects.filter( + bug_number__in=[bug_info["id"] for bug_info in bugs["bugs"]] + ) + + bug_status_map = {label: value for value, label in PerformanceAlertSummary.BUG_STATUSES} + bugs_by_id = {bug["id"]: bug for bug in bugs["bugs"]} + updates = {} + summaries_to_update = {} + for summary in alert_summaries: + bug = bugs_by_id.get(summary.bug_number) + if not bug: + continue + new_bug_status = bug_status_map.get(bug["resolution"]) + if new_bug_status is None and bug["resolution"] == "": + new_bug_status = PerformanceAlertSummary.BUG_NEW + if new_bug_status is None: + continue + if summary.bug_status == new_bug_status: + continue + updates[str(summary.id)] = {"bug_status": new_bug_status} + summaries_to_update[str(summary.id)] = summary + + return updates, summaries_to_update diff --git a/treeherder/perf/auto_perf_sheriffing/sherlock.py b/treeherder/perf/auto_perf_sheriffing/sherlock.py index 4451c04d222..3ad41335532 100644 --- a/treeherder/perf/auto_perf_sheriffing/sherlock.py +++ b/treeherder/perf/auto_perf_sheriffing/sherlock.py @@ -14,6 +14,9 @@ BackfillReportMaintainer, ) from treeherder.perf.auto_perf_sheriffing.backfill_tool import BackfillTool +from treeherder.perf.auto_perf_sheriffing.performance_alerting.alert_manager import ( + PerformanceAlertManager, +) from treeherder.perf.auto_perf_sheriffing.secretary import Secretary from treeherder.perf.auto_perf_sheriffing.telemetry_alerting.alert import ( TelemetryAlertFactory, @@ -99,7 +102,7 @@ def sheriff(self, since: datetime, frameworks: list[str], repositories: list[str self.assert_can_run() # reporter tool should always run *(only handles preliminary records/reports)* - logger.info("Sherlock: Reporter tool is creating/maintaining reports...") + logger.info("Sherlock: Reporter tool is creating/maintaining reports...") self._report(since, frameworks, repositories) self.assert_can_run() @@ -108,6 +111,9 @@ def sheriff(self, since: datetime, frameworks: list[str], repositories: list[str self._backfill(frameworks, repositories) self.assert_can_run() + logger.info("Sherlock: Syncing performance alert summary statuses...") + self._sync_performance_alert_summary_status() + def runtime_exceeded(self) -> bool: elapsed_runtime = datetime.now() - self._wake_up_time return self._max_runtime <= elapsed_runtime @@ -252,6 +258,10 @@ def __get_data_points_to_backfill(context: list[dict]) -> list[dict]: return context[start:] + def _sync_performance_alert_summary_status(self): + alert_manager = PerformanceAlertManager() + alert_manager.manage_alerts([]) + def telemetry_alert(self): if not self._can_run_telemetry(): return diff --git a/treeherder/perf/migrations/0069_performancealertsummary_bug_status.py b/treeherder/perf/migrations/0069_performancealertsummary_bug_status.py new file mode 100644 index 00000000000..a47e3b57995 --- /dev/null +++ b/treeherder/perf/migrations/0069_performancealertsummary_bug_status.py @@ -0,0 +1,35 @@ +# Generated by Django 5.1.15 on 2026-04-15 12:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "perf", + "0064_performancealerttesting_confidences_squashed_0068_remove_performancealerttesting_detected_changes_and_more", + ), + ] + + operations = [ + migrations.AddField( + model_name="performancealertsummary", + name="bug_status", + field=models.IntegerField( + choices=[ + (0, "NEW"), + (1, "FIXED"), + (2, "INVALID"), + (3, "INACTIVE"), + (4, "DUPLICATE"), + (5, "WONTFIX"), + (6, "WORKSFORME"), + (7, "INCOMPLETE"), + (8, "MOVED"), + ], + default=None, + null=True, + ), + ), + ] diff --git a/treeherder/perf/models.py b/treeherder/perf/models.py index b596ecfde57..f3e98d73263 100644 --- a/treeherder/perf/models.py +++ b/treeherder/perf/models.py @@ -493,6 +493,29 @@ def __str__(self): class PerformanceAlertSummary(PerformanceAlertSummaryBase): + BUG_NEW = 0 + BUG_FIXED = 1 + BUG_INVALID = 2 + BUG_INACTIVE = 3 + BUG_DUPLICATE = 4 + BUG_WONTFIX = 5 + BUG_WORKSFORME = 6 + BUG_INCOMPLETE = 7 + BUG_MOVED = 8 + + BUG_STATUSES = ( + (BUG_NEW, "NEW"), + (BUG_FIXED, "FIXED"), + (BUG_INVALID, "INVALID"), + (BUG_INACTIVE, "INACTIVE"), + (BUG_DUPLICATE, "DUPLICATE"), + (BUG_WONTFIX, "WONTFIX"), + (BUG_WORKSFORME, "WORKSFORME"), + (BUG_INCOMPLETE, "INCOMPLETE"), + (BUG_MOVED, "MOVED"), + ) + bug_status = models.IntegerField(choices=BUG_STATUSES, null=True, default=None) + class Meta: db_table = "performance_alert_summary" unique_together = ("repository", "framework", "prev_push", "push", "sheriffed")