Add latest alert to public api alert groups endpoint (grafana#5059)

# What this PR does

Added last alert information 
and optimized the API call so it makes 10x less queries by:
* prefetching chatops messages (based on @vadimkerr 's
grafana#4738)
* using `enrich` from private api

Previously: 
<img width="1102" alt="Screenshot 2024-09-24 at 4 47 00 PM"
src="https://github.com/user-attachments/assets/84edb78e-257a-49cd-bc94-083dd8d043d7">
Now:
<img width="1066" alt="Screenshot 2024-09-24 at 4 44 56 PM"
src="https://github.com/user-attachments/assets/e7dfcc40-dce6-4a0d-9677-910aab2b4f17">



## Which issue(s) this PR closes

Related to [issue link here]

<!--
*Note*: If you want the issue to be auto-closed once the PR is merged,
change "Related to" to "Closes" in the line above.
If you have more than one GitHub issue that this PR closes, be sure to
preface
each issue link with a [closing
keyword](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/using-keywords-in-issues-and-pull-requests#linking-a-pull-request-to-an-issue).
This ensures that the issue(s) are auto-closed once the PR has been
merged.
-->

## Checklist

- [ ] Unit, integration, and e2e (if applicable) tests updated
- [ ] Documentation added (or `pr:no public docs` PR label added if not
required)
- [ ] Added the relevant release notes label (see labels prefixed w/
`release:`). These labels dictate how your PR will
    show up in the autogenerated release notes.

---------

Co-authored-by: Vadim Stepanov <[email protected]>

docs/sources/oncall-api-reference/alertgroups.md

@@ -46,6 +46,31 @@ The above command returns JSON structured in the following way:
         "telegram": "https://t.me/c/5354/1234?thread=1234"
       },
       "silenced_at": "2020-05-19T13:37:01.429805Z",
+      "last_alert": {
+        "id": "AA74DN7T4JQB6",
+        "alert_group_id": "I68T24C13IFW1",
+        "created_at": "2020-05-11T20:08:43Z",
+        "payload": {
+          "state": "alerting",
+          "title": "[Alerting] Test notification",
+          "ruleId": 0,
+          "message": "Someone is testing the alert notification within Grafana.",
+          "ruleUrl": "{{API_URL}}/",
+          "ruleName": "Test notification",
+          "evalMatches": [
+            {
+              "tags": null,
+              "value": 100,
+              "metric": "High value"
+            },
+            {
+              "tags": null,
+              "value": 200,
+              "metric": "Higher Value"
+            }
+          ]
+        }
+      },
     }
   ],
   "current_page_number": 1,

engine/apps/api/serializers/alert_group.py

@@ -2,7 +2,6 @@
 import logging
 import typing
 
-from django.conf import settings
 from django.core.cache import cache
 from django.db.models import Prefetch
 from django.utils import timezone
@@ -133,26 +132,23 @@ class AlertGroupListSerializer(
 
     labels = AlertGroupLabelSerializer(many=True, read_only=True)
 
-    PREFETCH_RELATED: list[str | Prefetch] = [
+    PREFETCH_RELATED = [
         "dependent_alert_groups",
         "log_records__author",
         "labels",
+        Prefetch(
+            "slack_messages",
+            queryset=SlackMessage.objects.select_related("_slack_team_identity").order_by("created_at")[:1],
+            to_attr="prefetched_slack_messages",
+        ),
+        Prefetch(
+            "telegram_messages",
+            queryset=TelegramMessage.objects.filter(
+                chat_id__startswith="-", message_type=TelegramMessage.ALERT_GROUP_MESSAGE
+            ).order_by("id")[:1],
+            to_attr="prefetched_telegram_messages",
+        ),
     ]
-    if settings.ALERT_GROUP_LIST_TRY_PREFETCH:
-        PREFETCH_RELATED += [
-            Prefetch(
-                "slack_messages",
-                queryset=SlackMessage.objects.select_related("_slack_team_identity").order_by("created_at")[:1],
-                to_attr="prefetched_slack_messages",
-            ),
-            Prefetch(
-                "telegram_messages",
-                queryset=TelegramMessage.objects.filter(
-                    chat_id__startswith="-", message_type=TelegramMessage.ALERT_GROUP_MESSAGE
-                ).order_by("id")[:1],
-                to_attr="prefetched_telegram_messages",
-            ),
-        ]
 
     SELECT_RELATED = [
         "channel__organization",

engine/apps/api/views/alert_group.py

@@ -1,9 +1,8 @@
-import typing
 from datetime import timedelta
 
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
-from django.db.models import Count, Max, Q
+from django.db.models import Q
 from django.utils import timezone
 from django_filters import rest_framework as filters
 from drf_spectacular.utils import extend_schema, inline_serializer
@@ -15,7 +14,7 @@
 from rest_framework.response import Response
 
 from apps.alerts.constants import ActionSource
-from apps.alerts.models import Alert, AlertGroup, AlertReceiveChannel, EscalationChain, ResolutionNote
+from apps.alerts.models import AlertGroup, AlertReceiveChannel, EscalationChain, ResolutionNote
 from apps.alerts.paging import unpage_user
 from apps.alerts.tasks import delete_alert_group, send_update_resolution_note_signal
 from apps.alerts.utils import is_declare_incident_step_enabled
@@ -37,7 +36,12 @@
     ModelFieldFilterMixin,
     MultipleChoiceCharFilter,
 )
-from common.api_helpers.mixins import PreviewTemplateMixin, PublicPrimaryKeyMixin, TeamFilteringMixin
+from common.api_helpers.mixins import (
+    AlertGroupEnrichingMixin,
+    PreviewTemplateMixin,
+    PublicPrimaryKeyMixin,
+    TeamFilteringMixin,
+)
 from common.api_helpers.paginators import AlertGroupCursorPaginator
 
 
@@ -257,6 +261,7 @@ def get_search_fields(self, view, request):
 
 
 class AlertGroupView(
+    AlertGroupEnrichingMixin,
     PreviewTemplateMixin,
     AlertGroupTeamFilteringMixin,
     PublicPrimaryKeyMixin[AlertGroup],
@@ -356,19 +361,8 @@ def get_queryset(self, ignore_filtering_by_available_teams=False):
                 labels__value_name=value,
             )
 
-        queryset = queryset.only("id")
-
         return queryset
 
-    def paginate_queryset(self, queryset):
-        """
-        All SQL joins (select_related and prefetch_related) will be performed AFTER pagination, so it only joins tables
-        for 25 alert groups, not the whole table.
-        """
-        alert_groups = super().paginate_queryset(queryset)
-        alert_groups = self.enrich(alert_groups)
-        return alert_groups
-
     def get_object(self):
         obj = super().get_object()
         obj = self.enrich([obj])[0]
@@ -434,48 +428,6 @@ def retrieve(self, request, *args, **kwargs):
         """
         return super().retrieve(request, *args, **kwargs)
 
-    def enrich(self, alert_groups: typing.List[AlertGroup]) -> typing.List[AlertGroup]:
-        """
-        This method performs select_related and prefetch_related (using setup_eager_loading) as well as in-memory joins
-        to add additional info like alert_count and last_alert for every alert group efficiently.
-        We need the last_alert because it's used by AlertGroupWebRenderer.
-        """
-
-        # enrich alert groups with select_related and prefetch_related
-        alert_group_pks = [alert_group.pk for alert_group in alert_groups]
-        queryset = AlertGroup.objects.filter(pk__in=alert_group_pks).order_by("-started_at")
-
-        queryset = self.get_serializer_class().setup_eager_loading(queryset)
-        alert_groups = list(queryset)
-
-        # get info on alerts count and last alert ID for every alert group
-        alerts_info = (
-            Alert.objects.values("group_id")
-            .filter(group_id__in=alert_group_pks)
-            .annotate(alerts_count=Count("group_id"), last_alert_id=Max("id"))
-        )
-        alerts_info_map = {info["group_id"]: info for info in alerts_info}
-
-        # fetch last alerts for every alert group
-        last_alert_ids = [info["last_alert_id"] for info in alerts_info_map.values()]
-        last_alerts = Alert.objects.filter(pk__in=last_alert_ids)
-        for alert in last_alerts:
-            # link group back to alert
-            alert.group = [alert_group for alert_group in alert_groups if alert_group.pk == alert.group_id][0]
-            alerts_info_map[alert.group_id].update({"last_alert": alert})
-
-        # add additional "alerts_count" and "last_alert" fields to every alert group
-        for alert_group in alert_groups:
-            try:
-                alert_group.last_alert = alerts_info_map[alert_group.pk]["last_alert"]
-                alert_group.alerts_count = alerts_info_map[alert_group.pk]["alerts_count"]
-            except KeyError:
-                # alert group has no alerts
-                alert_group.last_alert = None
-                alert_group.alerts_count = 0
-
-        return alert_groups
-
     def destroy(self, request, *args, **kwargs):
         instance = self.get_object()
         delete_alert_group.apply_async((instance.pk, request.user.pk))

engine/apps/public_api/serializers/alert_groups.py

@@ -1,7 +1,11 @@
+from django.db.models import Prefetch
 from rest_framework import serializers
 
 from apps.alerts.models import AlertGroup
 from apps.api.serializers.alert_group import AlertGroupLabelSerializer
+from apps.public_api.serializers.alerts import AlertSerializer
+from apps.slack.models import SlackMessage
+from apps.telegram.models import TelegramMessage
 from common.api_helpers.custom_fields import TeamPrimaryKeyRelatedField, UserIdField
 from common.api_helpers.mixins import EagerLoadingMixin
 
@@ -18,9 +22,31 @@ class AlertGroupSerializer(EagerLoadingMixin, serializers.ModelSerializer):
     acknowledged_by = UserIdField(read_only=True, source="acknowledged_by_user")
     resolved_by = UserIdField(read_only=True, source="resolved_by_user")
     labels = AlertGroupLabelSerializer(many=True, read_only=True)
+    last_alert = serializers.SerializerMethodField()
 
-    SELECT_RELATED = ["channel", "channel_filter", "slack_message", "channel__organization", "channel__team"]
-    PREFETCH_RELATED = ["labels"]
+    SELECT_RELATED = [
+        "channel",
+        "channel_filter",
+        "channel__organization",
+        "channel__team",
+        "acknowledged_by_user",
+        "resolved_by_user",
+    ]
+    PREFETCH_RELATED = [
+        "labels",
+        Prefetch(
+            "slack_messages",
+            queryset=SlackMessage.objects.select_related("_slack_team_identity").order_by("created_at")[:1],
+            to_attr="prefetched_slack_messages",
+        ),
+        Prefetch(
+            "telegram_messages",
+            queryset=TelegramMessage.objects.filter(
+                chat_id__startswith="-", message_type=TelegramMessage.ALERT_GROUP_MESSAGE
+            ).order_by("id")[:1],
+            to_attr="prefetched_telegram_messages",
+        ),
+    ]
 
     class Meta:
         model = AlertGroup
@@ -40,14 +66,12 @@ class Meta:
             "title",
             "permalinks",
             "silenced_at",
+            "last_alert",
         ]
 
     def get_title(self, obj):
         return obj.web_title_cache
 
-    def get_alerts_count(self, obj):
-        return obj.alerts.count()
-
     def get_state(self, obj):
         return obj.state
 
@@ -56,3 +80,20 @@ def get_route_id(self, obj):
             return obj.channel_filter.public_primary_key
         else:
             return None
+
+    def get_last_alert(self, obj):
+        if hasattr(obj, "last_alert"):  # could be set by AlertGroupEnrichingMixin.enrich
+            last_alert = obj.last_alert
+        else:
+            last_alert = obj.alerts.order_by("-created_at").first()
+
+        if last_alert is None:
+            return None
+
+        return AlertSerializer(last_alert).data
+
+    def get_alerts_count(self, obj):
+        if hasattr(obj, "alerts_count"):  # could be set by AlertGroupEnrichingMixin.enrich
+            return obj.alerts_count
+
+        return obj.alerts.count()

engine/apps/public_api/tests/test_alert_groups.py

@@ -71,6 +71,12 @@ def user_pk_or_none(alert_group, user_field):
                     "web": alert_group.web_link,
                 },
                 "silenced_at": silenced_at,
+                "last_alert": {
+                    "id": alert_group.alerts.last().public_primary_key,
+                    "alert_group_id": alert_group.public_primary_key,
+                    "created_at": alert_group.alerts.last().created_at.isoformat().replace("+00:00", "Z"),
+                    "payload": alert_group.channel.config.example_payload,
+                },
             }
         )
     return {
@@ -110,7 +116,7 @@ def alert_group_public_api_setup(
 
     make_alert(alert_group=grafana_alert_group_default_route, raw_request_data=grafana.config.example_payload)
     make_alert(alert_group=grafana_alert_group_non_default_route, raw_request_data=grafana.config.example_payload)
-    make_alert(alert_group=formatted_webhook_alert_group, raw_request_data=grafana.config.example_payload)
+    make_alert(alert_group=formatted_webhook_alert_group, raw_request_data=formatted_webhook.config.example_payload)
 
     integrations = grafana, formatted_webhook
     alert_groups = (

engine/apps/public_api/tests/test_escalation.py

@@ -73,6 +73,12 @@ def test_escalation_new_alert_group(
             "web": f"a/grafana-oncall-app/alert-groups/{ag.public_primary_key}",
         },
         "silenced_at": None,
+        "last_alert": {
+            "id": ag.alerts.last().public_primary_key,
+            "alert_group_id": ag.public_primary_key,
+            "created_at": ag.alerts.last().created_at.isoformat().replace("+00:00", "Z"),
+            "payload": ag.alerts.last().raw_request_data,
+        },
     }
 
     alert = ag.alerts.get()

engine/apps/public_api/views/alert_groups.py

@@ -8,7 +8,7 @@
 from rest_framework.viewsets import GenericViewSet
 
 from apps.alerts.constants import ActionSource
-from apps.alerts.models import AlertGroup
+from apps.alerts.models import AlertGroup, AlertReceiveChannel
 from apps.alerts.tasks import delete_alert_group, wipe
 from apps.api.label_filtering import parse_label_query
 from apps.auth_token.auth import ApiTokenAuthentication
@@ -23,7 +23,7 @@
     DateRangeFilterMixin,
     get_team_queryset,
 )
-from common.api_helpers.mixins import RateLimitHeadersMixin
+from common.api_helpers.mixins import AlertGroupEnrichingMixin, RateLimitHeadersMixin
 from common.api_helpers.paginators import FiftyPageSizePaginator
 
 
@@ -49,7 +49,12 @@ class AlertGroupFilters(ByTeamModelFieldFilterMixin, DateRangeFilterMixin, filte
 
 
 class AlertGroupView(
-    RateLimitHeadersMixin, mixins.ListModelMixin, mixins.RetrieveModelMixin, mixins.DestroyModelMixin, GenericViewSet
+    AlertGroupEnrichingMixin,
+    RateLimitHeadersMixin,
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    mixins.DestroyModelMixin,
+    GenericViewSet,
 ):
     authentication_classes = (ApiTokenAuthentication,)
     permission_classes = (IsAuthenticated,)
@@ -64,18 +69,23 @@ class AlertGroupView(
     filterset_class = AlertGroupFilters
 
     def get_queryset(self):
+        # no select_related or prefetch_related is used at this point, it will be done on paginate_queryset.
+
         route_id = self.request.query_params.get("route_id", None)
         integration_id = self.request.query_params.get("integration_id", None)
         state = self.request.query_params.get("state", None)
 
-        queryset = AlertGroup.objects.filter(
-            channel__organization=self.request.auth.organization,
-        ).order_by("-started_at")
+        alert_receive_channels_qs = AlertReceiveChannel.objects_with_deleted.filter(
+            organization_id=self.request.auth.organization.id
+        )
+        if integration_id:
+            alert_receive_channels_qs = alert_receive_channels_qs.filter(public_primary_key=integration_id)
+
+        alert_receive_channels_ids = list(alert_receive_channels_qs.values_list("id", flat=True))
+        queryset = AlertGroup.objects.filter(channel__in=alert_receive_channels_ids).order_by("-started_at")
 
         if route_id:
             queryset = queryset.filter(channel_filter__public_primary_key=route_id)
-        if integration_id:
-            queryset = queryset.filter(channel__public_primary_key=integration_id)
         if state:
             choices = dict(AlertGroup.STATUS_CHOICES)
             try:
@@ -112,9 +122,11 @@ def get_object(self):
         public_primary_key = self.kwargs["pk"]
 
         try:
-            return AlertGroup.objects.filter(
+            obj = AlertGroup.objects.filter(
                 channel__organization=self.request.auth.organization,
             ).get(public_primary_key=public_primary_key)
+            obj = self.enrich([obj])[0]
+            return obj
         except AlertGroup.DoesNotExist:
             raise NotFound