Skip to content

Commit bf5998f

Browse files
Maffoochclaudevalentijnscholten
authored
Dispatch create-path notifications async to fix slow POST latency (#14731)
* Dispatch create-path notifications async to fix slow POST latency POST /api/v2/engagements/ takes ~5s on large tenants because create_notification runs recipient enumeration and per-user Alert writes on the request thread. Move the outer create_notification to a Celery worker for the five create-path events (engagement_added, product_added, product_type_added, finding_added, test_added) by adding async_create_notification (accepts ids, re-fetches, delegates) and dispatching via dojo_dispatch_task. This extends the existing per-user async pattern (Slack/email/MSTeams/webhooks) up one level so the recipient query and Alert fan-out no longer block the response. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * Fix ruff D213 and skip dispatch during fixture loads - Reformat async_create_notification importer-guard docstring to D213 style - Skip post_save dispatch when raw=True (loaddata) so the k8s initializer's fixture install path doesn't require an available Celery broker. Without this guard the unconditional async dispatch tries to enqueue during product_type.json load and fails with kombu OperationalError. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * optimize async_create_notification to avoid redundant DB fetches When test_id + engagement_id + product_id are all passed, the original implementation fetched each object independently (3 queries). Since Test.select_related("engagement__product") already loads all three in one query, derive engagement and product from the test instead. Same for engagement_id + product_id: one query instead of two. Also updates the no_async performance test expected counts accordingly. --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-authored-by: valentijnscholten <valentijnscholten@gmail.com>
1 parent 2cfda15 commit bf5998f

9 files changed

Lines changed: 303 additions & 79 deletions

File tree

‎dojo/api_v2/serializers.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import dojo.risk_acceptance.helper as ra_helper
2929
from dojo.authorization.authorization import user_has_permission
3030
from dojo.authorization.roles_permissions import Permissions
31+
from dojo.celery_dispatch import dojo_dispatch_task
3132
from dojo.endpoint.utils import endpoint_filter, endpoint_meta_import
3233
from dojo.finding.helper import (
3334
save_endpoints_template,
@@ -116,7 +117,7 @@
116117
Vulnerability_Id,
117118
get_current_date,
118119
)
119-
from dojo.notifications.helper import create_notification
120+
from dojo.notifications.helper import async_create_notification
120121
from dojo.product_announcements import (
121122
LargeScanSizeProductAnnouncement,
122123
ScanTypeProductAnnouncement,
@@ -2086,10 +2087,11 @@ def create(self, validated_data):
20862087
jira_helper.push_to_jira(new_finding)
20872088

20882089
# Create a notification
2089-
create_notification(
2090+
dojo_dispatch_task(
2091+
async_create_notification,
20902092
event="finding_added",
20912093
title=_("Addition of %s") % new_finding.title,
2092-
finding=new_finding,
2094+
finding_id=new_finding.id,
20932095
description=_('Finding "%s" was added by %s') % (new_finding.title, new_finding.reporter),
20942096
url=reverse("view_finding", args=(new_finding.id,)),
20952097
icon="exclamation-triangle",

‎dojo/engagement/signals.py‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,19 +7,30 @@
77
from django.urls import reverse
88
from django.utils.translation import gettext as _
99

10+
from dojo.celery_dispatch import dojo_dispatch_task
1011
from dojo.file_uploads.helper import delete_related_files
1112
from dojo.models import Engagement, Product
1213
from dojo.notes.helper import delete_related_notes
13-
from dojo.notifications.helper import create_notification
14+
from dojo.notifications.helper import async_create_notification, create_notification
1415
from dojo.pghistory_models import DojoEvents
1516

1617

1718
@receiver(post_save, sender=Engagement)
1819
def engagement_post_save(sender, instance, created, **kwargs):
20+
# raw=True is set by loaddata; skip dispatch so fixture loading doesn't require a live broker.
21+
if kwargs.get("raw"):
22+
return
1923
if created:
2024
title = _('Engagement created for "%(product)s": %(name)s') % {"product": instance.product, "name": instance.name}
21-
create_notification(event="engagement_added", title=title, engagement=instance, product=instance.product,
22-
url=reverse("view_engagement", args=(instance.id,)), url_api=reverse("engagement-detail", args=(instance.id,)))
25+
dojo_dispatch_task(
26+
async_create_notification,
27+
event="engagement_added",
28+
title=title,
29+
engagement_id=instance.id,
30+
product_id=instance.product_id,
31+
url=reverse("view_engagement", args=(instance.id,)),
32+
url_api=reverse("engagement-detail", args=(instance.id,)),
33+
)
2334

2435

2536
@receiver(pre_save, sender=Engagement)

‎dojo/importers/default_importer.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
Test,
1919
Test_Import,
2020
)
21-
from dojo.notifications.helper import create_notification
21+
from dojo.notifications.helper import async_create_notification
2222
from dojo.utils import get_full_url, perform_product_grading
2323
from dojo.validators import clean_tags
2424

@@ -140,12 +140,13 @@ def process_scan(
140140
)
141141
# Send out some notifications to the user
142142
logger.debug("IMPORT_SCAN: Generating notifications")
143-
create_notification(
143+
dojo_dispatch_task(
144+
async_create_notification,
144145
event="test_added",
145146
title=f"Test created for {self.test.engagement.product}: {self.test.engagement.name}: {self.test}",
146-
test=self.test,
147-
engagement=self.test.engagement,
148-
product=self.test.engagement.product,
147+
test_id=self.test.id,
148+
engagement_id=self.test.engagement_id,
149+
product_id=self.test.engagement.product_id,
149150
url=reverse("view_test", args=(self.test.id,)),
150151
url_api=reverse("test-detail", args=(self.test.id,)),
151152
)

‎dojo/notifications/helper.py‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -911,6 +911,67 @@ def send_webhooks_notification(event: str, user_id: int | None = None, **kwargs:
911911
get_manager_class_instance()._get_manager_instance("webhooks").send_webhooks_notification(event, user=user, **kwargs)
912912

913913

914+
@app.task
915+
def async_create_notification(
916+
event: str,
917+
engagement_id: int | None = None,
918+
product_id: int | None = None,
919+
product_type_id: int | None = None,
920+
finding_id: int | None = None,
921+
test_id: int | None = None,
922+
**kwargs: dict,
923+
) -> None:
924+
# Re-fetch by id so the recipient-enumeration query and per-user Alert writes
925+
# run in the worker rather than the request thread.
926+
# Fetch most-specific first and derive parent objects from the already-loaded
927+
# select_related chain to avoid redundant queries. For example, fetching a
928+
# Test with select_related("engagement__product") covers all three objects in
929+
# one query, so engagement_id and product_id don't need separate fetches.
930+
fetched_engagement = None
931+
fetched_product = None
932+
933+
if test_id is not None:
934+
test = Test.objects.filter(pk=test_id).select_related("engagement__product").first()
935+
if test is None:
936+
return
937+
kwargs["test"] = test
938+
fetched_engagement = test.engagement
939+
fetched_product = test.engagement.product
940+
941+
if engagement_id is not None:
942+
if fetched_engagement is not None:
943+
kwargs["engagement"] = fetched_engagement
944+
else:
945+
engagement = Engagement.objects.filter(pk=engagement_id).select_related("product").first()
946+
if engagement is None:
947+
return
948+
kwargs["engagement"] = engagement
949+
fetched_product = engagement.product
950+
951+
if product_id is not None:
952+
if fetched_product is not None:
953+
kwargs["product"] = fetched_product
954+
else:
955+
product = Product.objects.filter(pk=product_id).first()
956+
if product is None:
957+
return
958+
kwargs["product"] = product
959+
960+
if product_type_id is not None:
961+
product_type = Product_Type.objects.filter(pk=product_type_id).first()
962+
if product_type is None:
963+
return
964+
kwargs["product_type"] = product_type
965+
966+
if finding_id is not None:
967+
finding = Finding.objects.filter(pk=finding_id).select_related("test__engagement__product").first()
968+
if finding is None:
969+
return
970+
kwargs["finding"] = finding
971+
972+
create_notification(event=event, **kwargs)
973+
974+
914975
@app.task(ignore_result=True)
915976
def webhook_reactivation(endpoint_id: int, **_kwargs: dict) -> None:
916977
get_manager_class_instance()._get_manager_instance("webhooks")._webhook_reactivation(endpoint_id=endpoint_id)

‎dojo/product/signals.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@
77
from django.urls import reverse
88
from django.utils.translation import gettext as _
99

10+
from dojo.celery_dispatch import dojo_dispatch_task
1011
from dojo.labels import get_labels
1112
from dojo.models import Product
12-
from dojo.notifications.helper import create_notification
13+
from dojo.notifications.helper import async_create_notification, create_notification
1314
from dojo.pghistory_models import DojoEvents
1415
from dojo.utils import get_current_user
1516

@@ -18,13 +19,18 @@
1819

1920
@receiver(post_save, sender=Product)
2021
def product_post_save(sender, instance, created, **kwargs):
22+
# raw=True is set by loaddata; skip dispatch so fixture loading doesn't require a live broker.
23+
if kwargs.get("raw"):
24+
return
2125
if created:
22-
create_notification(event="product_added",
23-
title=instance.name,
24-
product=instance,
25-
url=reverse("view_product", args=(instance.id,)),
26-
url_api=reverse("product-detail", args=(instance.id,)),
27-
)
26+
dojo_dispatch_task(
27+
async_create_notification,
28+
event="product_added",
29+
title=instance.name,
30+
product_id=instance.id,
31+
url=reverse("view_product", args=(instance.id,)),
32+
url_api=reverse("product-detail", args=(instance.id,)),
33+
)
2834

2935

3036
@receiver(post_delete, sender=Product)

‎dojo/product_type/signals.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,23 +8,29 @@
88
from django.urls import reverse
99
from django.utils.translation import gettext as _
1010

11+
from dojo.celery_dispatch import dojo_dispatch_task
1112
from dojo.labels import get_labels
1213
from dojo.models import Product_Type
13-
from dojo.notifications.helper import create_notification
14+
from dojo.notifications.helper import async_create_notification, create_notification
1415
from dojo.pghistory_models import DojoEvents
1516

1617
labels = get_labels()
1718

1819

1920
@receiver(post_save, sender=Product_Type)
2021
def product_type_post_save(sender, instance, created, **kwargs):
22+
# raw=True is set by loaddata; skip dispatch so fixture loading doesn't require a live broker.
23+
if kwargs.get("raw"):
24+
return
2125
if created:
22-
create_notification(event="product_type_added",
23-
title=instance.name,
24-
product_type=instance,
25-
url=reverse("view_product_type", args=(instance.id,)),
26-
url_api=reverse("product_type-detail", args=(instance.id,)),
27-
)
26+
dojo_dispatch_task(
27+
async_create_notification,
28+
event="product_type_added",
29+
title=instance.name,
30+
product_type_id=instance.id,
31+
url=reverse("view_product_type", args=(instance.id,)),
32+
url_api=reverse("product_type-detail", args=(instance.id,)),
33+
)
2834

2935

3036
@receiver(post_delete, sender=Product_Type)

‎unittests/test_importers_performance.py‎

Lines changed: 30 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -315,11 +315,13 @@ def test_import_reimport_reimport_performance_pghistory_async(self):
315315

316316
self._import_reimport_performance(
317317
expected_num_queries1=139,
318-
expected_num_async_tasks1=1,
318+
expected_num_async_tasks1=2,
319319
expected_num_queries2=115,
320320
expected_num_async_tasks2=1,
321321
expected_num_queries3=29,
322322
expected_num_async_tasks3=1,
323+
expected_num_queries4=100,
324+
expected_num_async_tasks4=0,
323325
)
324326

325327
@override_settings(ENABLE_AUDITLOG=True)
@@ -336,12 +338,14 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self):
336338
testuser.usercontactinfo.save()
337339

338340
self._import_reimport_performance(
339-
expected_num_queries1=146,
340-
expected_num_async_tasks1=1,
341+
expected_num_queries1=152,
342+
expected_num_async_tasks1=2,
341343
expected_num_queries2=122,
342344
expected_num_async_tasks2=1,
343345
expected_num_queries3=36,
344346
expected_num_async_tasks3=1,
347+
expected_num_queries4=100,
348+
expected_num_async_tasks4=0,
345349
)
346350

347351
@override_settings(ENABLE_AUDITLOG=True)
@@ -359,12 +363,14 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr
359363
self.system_settings(enable_product_grade=True)
360364

361365
self._import_reimport_performance(
362-
expected_num_queries1=153,
363-
expected_num_async_tasks1=3,
366+
expected_num_queries1=159,
367+
expected_num_async_tasks1=4,
364368
expected_num_queries2=129,
365369
expected_num_async_tasks2=3,
366370
expected_num_queries3=40,
367371
expected_num_async_tasks3=3,
372+
expected_num_queries4=107,
373+
expected_num_async_tasks4=2,
368374
)
369375

370376
# Deduplication is enabled in the tests above, but to properly test it we must run the same import twice and capture the results.
@@ -483,9 +489,9 @@ def test_deduplication_performance_pghistory_async(self):
483489

484490
self._deduplication_performance(
485491
expected_num_queries1=74,
486-
expected_num_async_tasks1=1,
487-
expected_num_queries2=69,
488-
expected_num_async_tasks2=1,
492+
expected_num_async_tasks1=2,
493+
expected_num_queries2=66,
494+
expected_num_async_tasks2=2,
489495
check_duplicates=False, # Async mode - deduplication happens later
490496
)
491497

@@ -503,10 +509,10 @@ def test_deduplication_performance_pghistory_no_async(self):
503509
testuser.usercontactinfo.save()
504510

505511
self._deduplication_performance(
506-
expected_num_queries1=81,
507-
expected_num_async_tasks1=1,
508-
expected_num_queries2=77,
509-
expected_num_async_tasks2=1,
512+
expected_num_queries1=87,
513+
expected_num_async_tasks1=2,
514+
expected_num_queries2=80,
515+
expected_num_async_tasks2=2,
510516
)
511517

512518

@@ -570,7 +576,7 @@ def test_import_reimport_reimport_performance_pghistory_async(self):
570576

571577
self._import_reimport_performance(
572578
expected_num_queries1=1191,
573-
expected_num_async_tasks1=6,
579+
expected_num_async_tasks1=7,
574580
expected_num_queries2=716,
575581
expected_num_async_tasks2=17,
576582
expected_num_queries3=346,
@@ -593,8 +599,8 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self):
593599
testuser.usercontactinfo.save()
594600

595601
self._import_reimport_performance(
596-
expected_num_queries1=1200,
597-
expected_num_async_tasks1=6,
602+
expected_num_queries1=1206,
603+
expected_num_async_tasks1=7,
598604
expected_num_queries2=725,
599605
expected_num_async_tasks2=17,
600606
expected_num_queries3=355,
@@ -618,8 +624,8 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr
618624
self.system_settings(enable_product_grade=True)
619625

620626
self._import_reimport_performance(
621-
expected_num_queries1=1210,
622-
expected_num_async_tasks1=8,
627+
expected_num_queries1=1216,
628+
expected_num_async_tasks1=9,
623629
expected_num_queries2=735,
624630
expected_num_async_tasks2=19,
625631
expected_num_queries3=359,
@@ -719,9 +725,9 @@ def test_deduplication_performance_pghistory_async(self):
719725

720726
self._deduplication_performance(
721727
expected_num_queries1=1411,
722-
expected_num_async_tasks1=7,
723-
expected_num_queries2=1016,
724-
expected_num_async_tasks2=7,
728+
expected_num_async_tasks1=8,
729+
expected_num_queries2=1013,
730+
expected_num_async_tasks2=8,
725731
check_duplicates=False, # Async mode - deduplication happens later
726732
)
727733

@@ -738,8 +744,8 @@ def test_deduplication_performance_pghistory_no_async(self):
738744
testuser.usercontactinfo.save()
739745

740746
self._deduplication_performance(
741-
expected_num_queries1=1420,
742-
expected_num_async_tasks1=7,
743-
expected_num_queries2=1132,
744-
expected_num_async_tasks2=7,
747+
expected_num_queries1=1426,
748+
expected_num_async_tasks1=8,
749+
expected_num_queries2=1135,
750+
expected_num_async_tasks2=8,
745751
)

0 commit comments

Comments
 (0)