diff --git a/docs/sources/oncall-api-reference/alertgroups.md b/docs/sources/oncall-api-reference/alertgroups.md index 9b9d007fd5..33c2d46b6b 100644 --- a/docs/sources/oncall-api-reference/alertgroups.md +++ b/docs/sources/oncall-api-reference/alertgroups.md @@ -46,6 +46,31 @@ The above command returns JSON structured in the following way: "telegram": "https://t.me/c/5354/1234?thread=1234" }, "silenced_at": "2020-05-19T13:37:01.429805Z", + "last_alert": { + "id": "AA74DN7T4JQB6", + "alert_group_id": "I68T24C13IFW1", + "created_at": "2020-05-11T20:08:43Z", + "payload": { + "state": "alerting", + "title": "[Alerting] Test notification", + "ruleId": 0, + "message": "Someone is testing the alert notification within Grafana.", + "ruleUrl": "{{API_URL}}/", + "ruleName": "Test notification", + "evalMatches": [ + { + "tags": null, + "value": 100, + "metric": "High value" + }, + { + "tags": null, + "value": 200, + "metric": "Higher Value" + } + ] + } + }, } ], "current_page_number": 1, diff --git a/engine/apps/api/serializers/alert_group.py b/engine/apps/api/serializers/alert_group.py index d4b2e65d5e..c0882658fb 100644 --- a/engine/apps/api/serializers/alert_group.py +++ b/engine/apps/api/serializers/alert_group.py @@ -2,7 +2,6 @@ import logging import typing -from django.conf import settings from django.core.cache import cache from django.db.models import Prefetch from django.utils import timezone @@ -133,26 +132,23 @@ class AlertGroupListSerializer( labels = AlertGroupLabelSerializer(many=True, read_only=True) - PREFETCH_RELATED: list[str | Prefetch] = [ + PREFETCH_RELATED = [ "dependent_alert_groups", "log_records__author", "labels", + Prefetch( + "slack_messages", + queryset=SlackMessage.objects.select_related("_slack_team_identity").order_by("created_at")[:1], + to_attr="prefetched_slack_messages", + ), + Prefetch( + "telegram_messages", + queryset=TelegramMessage.objects.filter( + chat_id__startswith="-", message_type=TelegramMessage.ALERT_GROUP_MESSAGE + ).order_by("id")[:1], + to_attr="prefetched_telegram_messages", + ), ] - if settings.ALERT_GROUP_LIST_TRY_PREFETCH: - PREFETCH_RELATED += [ - Prefetch( - "slack_messages", - queryset=SlackMessage.objects.select_related("_slack_team_identity").order_by("created_at")[:1], - to_attr="prefetched_slack_messages", - ), - Prefetch( - "telegram_messages", - queryset=TelegramMessage.objects.filter( - chat_id__startswith="-", message_type=TelegramMessage.ALERT_GROUP_MESSAGE - ).order_by("id")[:1], - to_attr="prefetched_telegram_messages", - ), - ] SELECT_RELATED = [ "channel__organization", diff --git a/engine/apps/api/views/alert_group.py b/engine/apps/api/views/alert_group.py index 8a24b877f3..5b158d1228 100644 --- a/engine/apps/api/views/alert_group.py +++ b/engine/apps/api/views/alert_group.py @@ -1,9 +1,8 @@ -import typing from datetime import timedelta from django.conf import settings from django.core.exceptions import ObjectDoesNotExist -from django.db.models import Count, Max, Q +from django.db.models import Q from django.utils import timezone from django_filters import rest_framework as filters from drf_spectacular.utils import extend_schema, inline_serializer @@ -15,7 +14,7 @@ from rest_framework.response import Response from apps.alerts.constants import ActionSource -from apps.alerts.models import Alert, AlertGroup, AlertReceiveChannel, EscalationChain, ResolutionNote +from apps.alerts.models import AlertGroup, AlertReceiveChannel, EscalationChain, ResolutionNote from apps.alerts.paging import unpage_user from apps.alerts.tasks import delete_alert_group, send_update_resolution_note_signal from apps.alerts.utils import is_declare_incident_step_enabled @@ -37,7 +36,12 @@ ModelFieldFilterMixin, MultipleChoiceCharFilter, ) -from common.api_helpers.mixins import PreviewTemplateMixin, PublicPrimaryKeyMixin, TeamFilteringMixin +from common.api_helpers.mixins import ( + AlertGroupEnrichingMixin, + PreviewTemplateMixin, + PublicPrimaryKeyMixin, + TeamFilteringMixin, +) from common.api_helpers.paginators import AlertGroupCursorPaginator @@ -257,6 +261,7 @@ def get_search_fields(self, view, request): class AlertGroupView( + AlertGroupEnrichingMixin, PreviewTemplateMixin, AlertGroupTeamFilteringMixin, PublicPrimaryKeyMixin[AlertGroup], @@ -356,19 +361,8 @@ def get_queryset(self, ignore_filtering_by_available_teams=False): labels__value_name=value, ) - queryset = queryset.only("id") - return queryset - def paginate_queryset(self, queryset): - """ - All SQL joins (select_related and prefetch_related) will be performed AFTER pagination, so it only joins tables - for 25 alert groups, not the whole table. - """ - alert_groups = super().paginate_queryset(queryset) - alert_groups = self.enrich(alert_groups) - return alert_groups - def get_object(self): obj = super().get_object() obj = self.enrich([obj])[0] @@ -434,48 +428,6 @@ def retrieve(self, request, *args, **kwargs): """ return super().retrieve(request, *args, **kwargs) - def enrich(self, alert_groups: typing.List[AlertGroup]) -> typing.List[AlertGroup]: - """ - This method performs select_related and prefetch_related (using setup_eager_loading) as well as in-memory joins - to add additional info like alert_count and last_alert for every alert group efficiently. - We need the last_alert because it's used by AlertGroupWebRenderer. - """ - - # enrich alert groups with select_related and prefetch_related - alert_group_pks = [alert_group.pk for alert_group in alert_groups] - queryset = AlertGroup.objects.filter(pk__in=alert_group_pks).order_by("-started_at") - - queryset = self.get_serializer_class().setup_eager_loading(queryset) - alert_groups = list(queryset) - - # get info on alerts count and last alert ID for every alert group - alerts_info = ( - Alert.objects.values("group_id") - .filter(group_id__in=alert_group_pks) - .annotate(alerts_count=Count("group_id"), last_alert_id=Max("id")) - ) - alerts_info_map = {info["group_id"]: info for info in alerts_info} - - # fetch last alerts for every alert group - last_alert_ids = [info["last_alert_id"] for info in alerts_info_map.values()] - last_alerts = Alert.objects.filter(pk__in=last_alert_ids) - for alert in last_alerts: - # link group back to alert - alert.group = [alert_group for alert_group in alert_groups if alert_group.pk == alert.group_id][0] - alerts_info_map[alert.group_id].update({"last_alert": alert}) - - # add additional "alerts_count" and "last_alert" fields to every alert group - for alert_group in alert_groups: - try: - alert_group.last_alert = alerts_info_map[alert_group.pk]["last_alert"] - alert_group.alerts_count = alerts_info_map[alert_group.pk]["alerts_count"] - except KeyError: - # alert group has no alerts - alert_group.last_alert = None - alert_group.alerts_count = 0 - - return alert_groups - def destroy(self, request, *args, **kwargs): instance = self.get_object() delete_alert_group.apply_async((instance.pk, request.user.pk)) diff --git a/engine/apps/public_api/serializers/alert_groups.py b/engine/apps/public_api/serializers/alert_groups.py index 07f1d0fbc6..5218bd1305 100644 --- a/engine/apps/public_api/serializers/alert_groups.py +++ b/engine/apps/public_api/serializers/alert_groups.py @@ -1,7 +1,11 @@ +from django.db.models import Prefetch from rest_framework import serializers from apps.alerts.models import AlertGroup from apps.api.serializers.alert_group import AlertGroupLabelSerializer +from apps.public_api.serializers.alerts import AlertSerializer +from apps.slack.models import SlackMessage +from apps.telegram.models import TelegramMessage from common.api_helpers.custom_fields import TeamPrimaryKeyRelatedField, UserIdField from common.api_helpers.mixins import EagerLoadingMixin @@ -18,9 +22,31 @@ class AlertGroupSerializer(EagerLoadingMixin, serializers.ModelSerializer): acknowledged_by = UserIdField(read_only=True, source="acknowledged_by_user") resolved_by = UserIdField(read_only=True, source="resolved_by_user") labels = AlertGroupLabelSerializer(many=True, read_only=True) + last_alert = serializers.SerializerMethodField() - SELECT_RELATED = ["channel", "channel_filter", "slack_message", "channel__organization", "channel__team"] - PREFETCH_RELATED = ["labels"] + SELECT_RELATED = [ + "channel", + "channel_filter", + "channel__organization", + "channel__team", + "acknowledged_by_user", + "resolved_by_user", + ] + PREFETCH_RELATED = [ + "labels", + Prefetch( + "slack_messages", + queryset=SlackMessage.objects.select_related("_slack_team_identity").order_by("created_at")[:1], + to_attr="prefetched_slack_messages", + ), + Prefetch( + "telegram_messages", + queryset=TelegramMessage.objects.filter( + chat_id__startswith="-", message_type=TelegramMessage.ALERT_GROUP_MESSAGE + ).order_by("id")[:1], + to_attr="prefetched_telegram_messages", + ), + ] class Meta: model = AlertGroup @@ -40,14 +66,12 @@ class Meta: "title", "permalinks", "silenced_at", + "last_alert", ] def get_title(self, obj): return obj.web_title_cache - def get_alerts_count(self, obj): - return obj.alerts.count() - def get_state(self, obj): return obj.state @@ -56,3 +80,20 @@ def get_route_id(self, obj): return obj.channel_filter.public_primary_key else: return None + + def get_last_alert(self, obj): + if hasattr(obj, "last_alert"): # could be set by AlertGroupEnrichingMixin.enrich + last_alert = obj.last_alert + else: + last_alert = obj.alerts.order_by("-created_at").first() + + if last_alert is None: + return None + + return AlertSerializer(last_alert).data + + def get_alerts_count(self, obj): + if hasattr(obj, "alerts_count"): # could be set by AlertGroupEnrichingMixin.enrich + return obj.alerts_count + + return obj.alerts.count() diff --git a/engine/apps/public_api/tests/test_alert_groups.py b/engine/apps/public_api/tests/test_alert_groups.py index 758b0d9924..71421cd318 100644 --- a/engine/apps/public_api/tests/test_alert_groups.py +++ b/engine/apps/public_api/tests/test_alert_groups.py @@ -71,6 +71,12 @@ def user_pk_or_none(alert_group, user_field): "web": alert_group.web_link, }, "silenced_at": silenced_at, + "last_alert": { + "id": alert_group.alerts.last().public_primary_key, + "alert_group_id": alert_group.public_primary_key, + "created_at": alert_group.alerts.last().created_at.isoformat().replace("+00:00", "Z"), + "payload": alert_group.channel.config.example_payload, + }, } ) return { @@ -110,7 +116,7 @@ def alert_group_public_api_setup( make_alert(alert_group=grafana_alert_group_default_route, raw_request_data=grafana.config.example_payload) make_alert(alert_group=grafana_alert_group_non_default_route, raw_request_data=grafana.config.example_payload) - make_alert(alert_group=formatted_webhook_alert_group, raw_request_data=grafana.config.example_payload) + make_alert(alert_group=formatted_webhook_alert_group, raw_request_data=formatted_webhook.config.example_payload) integrations = grafana, formatted_webhook alert_groups = ( diff --git a/engine/apps/public_api/tests/test_escalation.py b/engine/apps/public_api/tests/test_escalation.py index 0fad2f71ee..5c4e0e772a 100644 --- a/engine/apps/public_api/tests/test_escalation.py +++ b/engine/apps/public_api/tests/test_escalation.py @@ -73,6 +73,12 @@ def test_escalation_new_alert_group( "web": f"a/grafana-oncall-app/alert-groups/{ag.public_primary_key}", }, "silenced_at": None, + "last_alert": { + "id": ag.alerts.last().public_primary_key, + "alert_group_id": ag.public_primary_key, + "created_at": ag.alerts.last().created_at.isoformat().replace("+00:00", "Z"), + "payload": ag.alerts.last().raw_request_data, + }, } alert = ag.alerts.get() diff --git a/engine/apps/public_api/views/alert_groups.py b/engine/apps/public_api/views/alert_groups.py index c1b5fe1d54..738219d428 100644 --- a/engine/apps/public_api/views/alert_groups.py +++ b/engine/apps/public_api/views/alert_groups.py @@ -8,7 +8,7 @@ from rest_framework.viewsets import GenericViewSet from apps.alerts.constants import ActionSource -from apps.alerts.models import AlertGroup +from apps.alerts.models import AlertGroup, AlertReceiveChannel from apps.alerts.tasks import delete_alert_group, wipe from apps.api.label_filtering import parse_label_query from apps.auth_token.auth import ApiTokenAuthentication @@ -23,7 +23,7 @@ DateRangeFilterMixin, get_team_queryset, ) -from common.api_helpers.mixins import RateLimitHeadersMixin +from common.api_helpers.mixins import AlertGroupEnrichingMixin, RateLimitHeadersMixin from common.api_helpers.paginators import FiftyPageSizePaginator @@ -49,7 +49,12 @@ class AlertGroupFilters(ByTeamModelFieldFilterMixin, DateRangeFilterMixin, filte class AlertGroupView( - RateLimitHeadersMixin, mixins.ListModelMixin, mixins.RetrieveModelMixin, mixins.DestroyModelMixin, GenericViewSet + AlertGroupEnrichingMixin, + RateLimitHeadersMixin, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + mixins.DestroyModelMixin, + GenericViewSet, ): authentication_classes = (ApiTokenAuthentication,) permission_classes = (IsAuthenticated,) @@ -64,18 +69,23 @@ class AlertGroupView( filterset_class = AlertGroupFilters def get_queryset(self): + # no select_related or prefetch_related is used at this point, it will be done on paginate_queryset. + route_id = self.request.query_params.get("route_id", None) integration_id = self.request.query_params.get("integration_id", None) state = self.request.query_params.get("state", None) - queryset = AlertGroup.objects.filter( - channel__organization=self.request.auth.organization, - ).order_by("-started_at") + alert_receive_channels_qs = AlertReceiveChannel.objects_with_deleted.filter( + organization_id=self.request.auth.organization.id + ) + if integration_id: + alert_receive_channels_qs = alert_receive_channels_qs.filter(public_primary_key=integration_id) + + alert_receive_channels_ids = list(alert_receive_channels_qs.values_list("id", flat=True)) + queryset = AlertGroup.objects.filter(channel__in=alert_receive_channels_ids).order_by("-started_at") if route_id: queryset = queryset.filter(channel_filter__public_primary_key=route_id) - if integration_id: - queryset = queryset.filter(channel__public_primary_key=integration_id) if state: choices = dict(AlertGroup.STATUS_CHOICES) try: @@ -112,9 +122,11 @@ def get_object(self): public_primary_key = self.kwargs["pk"] try: - return AlertGroup.objects.filter( + obj = AlertGroup.objects.filter( channel__organization=self.request.auth.organization, ).get(public_primary_key=public_primary_key) + obj = self.enrich([obj])[0] + return obj except AlertGroup.DoesNotExist: raise NotFound diff --git a/engine/common/api_helpers/mixins.py b/engine/common/api_helpers/mixins.py index d6e31c4261..552aaade45 100644 --- a/engine/common/api_helpers/mixins.py +++ b/engine/common/api_helpers/mixins.py @@ -4,7 +4,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.db import models -from django.db.models import Q +from django.db.models import Count, Max, Q from django.utils.functional import cached_property from drf_spectacular.utils import extend_schema, inline_serializer from rest_framework import serializers, status @@ -21,6 +21,7 @@ AlertWebTemplater, TemplateLoader, ) +from apps.alerts.models import Alert, AlertGroup from apps.base.messaging import get_messaging_backends from common.api_helpers.exceptions import BadRequest from common.jinja_templater import apply_jinja_template @@ -411,3 +412,56 @@ def instance_context(self) -> InstanceContext: else: instance_context = None return instance_context + + +class AlertGroupEnrichingMixin: + def paginate_queryset(self, queryset): + """ + All SQL joins (select_related and prefetch_related) will be performed AFTER pagination, so it only joins tables + for one page of alert groups, not the whole table. + """ + alert_groups = super().paginate_queryset(queryset.only("id")) + alert_groups = self.enrich(alert_groups) + return alert_groups + + def enrich(self, alert_groups: typing.List[AlertGroup]) -> typing.List[AlertGroup]: + """ + This method performs select_related and prefetch_related (using setup_eager_loading) as well as in-memory joins + to add additional info like alert_count and last_alert for every alert group efficiently. + We need the last_alert because it's used by AlertGroupWebRenderer. + """ + + # enrich alert groups with select_related and prefetch_related + alert_group_pks = [alert_group.pk for alert_group in alert_groups] + queryset = AlertGroup.objects.filter(pk__in=alert_group_pks).order_by("-started_at") + + queryset = self.get_serializer_class().setup_eager_loading(queryset) + alert_groups = list(queryset) + + # get info on alerts count and last alert ID for every alert group + alerts_info = ( + Alert.objects.values("group_id") + .filter(group_id__in=alert_group_pks) + .annotate(alerts_count=Count("group_id"), last_alert_id=Max("id")) + ) + alerts_info_map = {info["group_id"]: info for info in alerts_info} + + # fetch last alerts for every alert group + last_alert_ids = [info["last_alert_id"] for info in alerts_info_map.values()] + last_alerts = Alert.objects.filter(pk__in=last_alert_ids) + for alert in last_alerts: + # link group back to alert + alert.group = [alert_group for alert_group in alert_groups if alert_group.pk == alert.group_id][0] + alerts_info_map[alert.group_id].update({"last_alert": alert}) + + # add additional "alerts_count" and "last_alert" fields to every alert group + for alert_group in alert_groups: + try: + alert_group.last_alert = alerts_info_map[alert_group.pk]["last_alert"] + alert_group.alerts_count = alerts_info_map[alert_group.pk]["alerts_count"] + except KeyError: + # alert group has no alerts + alert_group.last_alert = None + alert_group.alerts_count = 0 + + return alert_groups diff --git a/engine/settings/base.py b/engine/settings/base.py index 4fd51a7b3d..c3b1a3971f 100644 --- a/engine/settings/base.py +++ b/engine/settings/base.py @@ -207,7 +207,6 @@ class DatabaseTypes: ALERT_GROUPS_DISABLE_PREFER_ORDERING_INDEX = DATABASE_TYPE == DatabaseTypes.MYSQL and getenv_boolean( "ALERT_GROUPS_DISABLE_PREFER_ORDERING_INDEX", default=False ) -ALERT_GROUP_LIST_TRY_PREFETCH = getenv_boolean("ALERT_GROUP_LIST_TRY_PREFETCH", default=False) # Redis REDIS_USERNAME = os.getenv("REDIS_USERNAME", "")