more responseive detection of 'rate-limited' crawls: (#3521)

- detect when all crawl pods are in a CrashBackLoop or Terminated status
with rate limit exit code (18) and set state to 'rate-limited'
- allows for rate limited state to be correctly conveyed in scaled
crawls where crawlers are still in exponential backoff loop while redis
key has expired

Author: ikreymer

backend/btrixcloud/operator/crawls.py

@@ -1236,6 +1236,8 @@ def sync_pod_status(
         redis_running = False
         pod_done_count = 0
 
+        all_crashed = True
+
         try:
             for name, pod in pods.items():
                 running = False
@@ -1250,22 +1252,39 @@ def sync_pod_status(
                 elif phase == "Failed" and pstatus.get("reason") == "Evicted":
                     evicted = True
 
-                status.podStatus[name].evicted = evicted
+                pod_status = status.podStatus[name]
+                pod_status.evicted = evicted
 
                 if "containerStatuses" in pstatus:
                     cstatus = pstatus["containerStatuses"][0]
 
-                    self.handle_terminated_pod(
-                        name, role, status, cstatus["state"].get("terminated")
+                    terminated = cstatus["state"].get("terminated")
+
+                    self.handle_terminated_pod(name, role, status, terminated)
+
+                    waiting = cstatus["state"].get("waiting")
+                    pod_status.backoffWait = (
+                        waiting and waiting.get("reason") == "CrashLoopBackOff"
                     )
 
+                    if role == "crawler":
+                        # consider crashed if:
+                        # - in 'waiting' state with CrashLoopBackOff reason
+                        # - in 'terminated' state with non-zero exit code (will be brief)
+                        all_crashed = all_crashed and bool(
+                            pod_status.backoffWait
+                            or (terminated and pod_status.exitCode != 0)
+                        )
+
                 if role == "crawler":
                     crawler_running = crawler_running or running
                     if phase == "Succeeded":
                         pod_done_count += 1
                 elif role == "redis":
                     redis_running = redis_running or running
 
+            status.allCrashed = all_crashed
+
         # pylint: disable=broad-except
         except Exception:
             logger.exception(
@@ -1294,13 +1313,14 @@ def handle_terminated_pod(
 
         pod_status = status.podStatus[name]
 
+        # detect reason
+        exit_code = terminated.get("exitCode")
+
         pod_status.isNewExit = pod_status.exitTime != exit_time
         if pod_status.isNewExit and role == "crawler":
             pod_status.exitTime = exit_time
             status.anyCrawlPodNewExit = True
-
-        # detect reason
-        exit_code = terminated.get("exitCode")
+            status.lastCrawlPodExitCode = exit_code
 
         if exit_code == 0:
             pod_status.reason = "done"
@@ -1877,6 +1897,21 @@ async def update_crawl_state(
             status.stopReason = await self.is_crawl_stopping(crawl, status, stats)
             status.stopping = status.stopReason is not None
 
+        # if all crashed and last exit was rate-limit exit code (18),
+        # set to rate limited state now and return
+        if status.allCrashed and status.lastCrawlPodExitCode == 18:
+            if not status.rateLimitedAtTime:
+                status.rateLimitedAtTime = date_to_str(dt_now())
+
+            await self.set_state(
+                "rate-limited",
+                status,
+                crawl,
+                allowed_from=RUNNING_STATES,
+            )
+            status.resync_after = self.fast_retry_secs
+            return status
+
         # mark crawl as pausing or stopping
         if status.stopping:
             if status.stopReason in PAUSED_STATES:

backend/btrixcloud/operator/models.py

@@ -159,6 +159,7 @@ class PodInfo(BaseModel):
     signalAtMem: int | None = None
 
     evicted: bool | None = False
+    backoffWait: bool | None = False
 
     lastWorkers: int | None = 0
 
@@ -275,6 +276,12 @@ class CrawlStatus(BaseModel):
     # any pods exited
     anyCrawlPodNewExit: bool | None = Field(default=False, exclude=True)
 
+    # exit code of last pod to exit
+    lastCrawlPodExitCode: int | None = 0
+
+    # all running pods have crashed
+    allCrashed: bool = False
+
     # if status is 'rate-limited', when first became rate-limited
     rateLimitedAtTime: str | None = None