optimization: drop low priority duplicate request fetch (#1157)

If a fetch request is low priority (most likely coming from behaviors
via webrecorder/browsertrix-behaviors#170) and it is a duplicate, abort
the request from being made altogether.

Fixes #1156

Author: ikreymer

src/util/recorder.ts

@@ -232,7 +232,10 @@ export class Recorder extends EventEmitter {
     });
 
     await cdp.send("Fetch.enable", {
-      patterns: [{ urlPattern: "*", requestStage: "Response" }],
+      patterns: [
+        { urlPattern: "*", requestStage: "Response" },
+        { requestStage: "Request", resourceType: "Fetch" },
+      ],
     });
 
     // Response
@@ -668,6 +671,14 @@ export class Recorder extends EventEmitter {
 
     let continued = false;
 
+    // response is either responseStatusCode or responseErrorReason are set
+    const isResponse = responseStatusCode || responseErrorReason;
+
+    if (!isResponse && networkId) {
+      await this.handleFetchRequest(networkId, requestId, cdp);
+      return;
+    }
+
     try {
       if (
         responseStatusCode &&
@@ -709,6 +720,37 @@ export class Recorder extends EventEmitter {
     }
   }
 
+  async handleFetchRequest(
+    networkId: string,
+    requestId: string,
+    cdp: CDPSession,
+  ) {
+    const reqresp = this.pendingReqResp(networkId, true);
+
+    if (
+      reqresp &&
+      reqresp.priority === "Low" &&
+      reqresp.resourceType === "fetch"
+    ) {
+      if (await this.isDupeFetch(reqresp)) {
+        this.removeReqResp(networkId);
+        await cdp.send("Fetch.failRequest", {
+          requestId,
+          errorReason: "Aborted",
+        });
+        logger.debug(
+          "Aborted dupe low-priority fetch in request phase",
+          { url: reqresp.url },
+          "recorder",
+        );
+        return false;
+      }
+    }
+
+    await cdp.send("Fetch.continueRequest", { requestId });
+    return true;
+  }
+
   async handleFetchResponse(
     params: Protocol.Fetch.RequestPausedEvent,
     cdp: CDPSession,

src/util/reqresp.ts

@@ -43,6 +43,8 @@ export class RequestResponseInfo {
   postData?: string;
   hasPostData: boolean = false;
 
+  priority?: string;
+
   // content-encoding is preserved, eg. payload is still encoded
   cePreserved = false;
 
@@ -127,6 +129,7 @@ export class RequestResponseInfo {
     if (resourceType) {
       this.resourceType = resourceType.toLowerCase();
     }
+    this.priority = request.initialPriority;
   }
 
   fillResponse(response: Protocol.Network.Response, resourceType?: string) {