Fix --alwaysAddBehaviorLinks / smart scoping to work as expected: (#1094)

- ensure regular links always obey scope, as using same method
- support addLink(url, alwaysObeyScope) from added in,
webrecorder/browsertrix-behaviors#150
- scope only ignored if alwaysObeyScope is false and
alwaysAddBehaviorLinks is set
- move addLink() declaration to setupPage(), as extractLinks() not
called for single page scope
- simplify, no longer need extra 'callbacks' in PageState

Author: ikreymer

src/crawler.ts

@@ -814,7 +814,25 @@ export class Crawler {
   }
 
   async setupPage(opts: WorkerState) {
-    const { page, cdp, workerid, callbacks, frameIdToExecId, recorder } = opts;
+    const { page, cdp, workerid, frameIdToExecId, recorder } = opts;
+
+    const addLink = async (url: string, alwaysObeyScope: boolean = false) => {
+      const { seedId, depth, extraHops = 0, logDetails } = opts.data;
+
+      // if not always obeying scope and allowed to ignore scope, set to true
+      const ignoreScope =
+        !alwaysObeyScope && this.params.alwaysAddBehaviorLinks;
+
+      await this.queueInScopeUrls({
+        seedId,
+        urls: [url],
+        depth,
+        extraHops,
+        pageUrl: page.url(),
+        logDetails,
+        ignoreScope,
+      });
+    };
 
     await this.browser.setupPage({ page, cdp });
 
@@ -865,10 +883,7 @@ export class Crawler {
       await this.screencaster.screencastPage(page, cdp, workerid);
     }
 
-    await page.exposeFunction(
-      BxFunctionBindings.AddLinkFunc,
-      (url: string) => callbacks.addLink && callbacks.addLink(url),
-    );
+    await page.exposeFunction(BxFunctionBindings.AddLinkFunc, addLink);
 
     // used for both behaviors and link extraction now
     await this.browser.addInitScript(page, btrixBehaviors);
@@ -999,7 +1014,7 @@ self.__bx_behaviors.selectMainBehavior();
       BxFunctionBindings.AddToSeenSet,
       (data: string) => {
         if (data && (data.startsWith("https:") || data.startsWith("http:"))) {
-          void callbacks.addLink(data);
+          void addLink(data);
         }
         return this.crawlState.addToUserSet(data);
       },
@@ -1122,8 +1137,7 @@ self.__bx_behaviors.selectMainBehavior();
   async crawlPage(opts: WorkerState): Promise<void> {
     await this.writeStats();
 
-    const { page, cdp, data, workerid, callbacks, recorder } = opts;
-    data.callbacks = callbacks;
+    const { page, cdp, data, workerid, recorder } = opts;
 
     const { url, seedId, depth } = data;
 
@@ -2713,25 +2727,7 @@ self.__bx_behaviors.selectMainBehavior();
     selectors: ExtractSelector[],
     logDetails: LogDetails,
   ) {
-    const { seedId, depth, extraHops = 0, filteredFrames, callbacks } = data;
-
-    callbacks.addLink = async (url: string, ignoreScope = false) => {
-      // if crawler arg set, always ignore scope
-      // otherwise, may be determined by behavior addLink()
-      if (this.params.alwaysAddBehaviorLinks) {
-        ignoreScope = true;
-      }
-
-      await this.queueInScopeUrls({
-        seedId,
-        urls: [url],
-        depth,
-        extraHops,
-        pageUrl: page.url(),
-        logDetails,
-        ignoreScope,
-      });
-    };
+    const { filteredFrames } = data;
 
     const frames = filteredFrames || page.frames();
 

src/util/worker.ts

@@ -19,8 +19,6 @@ export type WorkerState = {
   page: Page;
   cdp: CDPSession;
   workerid: WorkerId;
-  // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
-  callbacks: Record<string, Function>;
   recorder: Recorder | null;
   markPageUsed: () => void;
   frameIdToExecId: Map<string, number>;
@@ -41,9 +39,6 @@ export class PageWorker {
   page?: Page | null;
   cdp?: CDPSession | null;
 
-  // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
-  callbacks?: Record<string, Function>;
-
   opts?: WorkerState;
 
   // TODO: Fix this the next time the file is edited.
@@ -171,13 +166,11 @@ export class PageWorker {
 
         this.page = page;
         this.cdp = cdp;
-        this.callbacks = {};
 
         this.opts = {
           page,
           cdp,
           workerid,
-          callbacks: this.callbacks,
           recorder: this.recorder,
           frameIdToExecId: new Map<string, number>(),
           markPageUsed: () => {