feat: add --stop-on-rate-limit and --retry-on-rate-limit flags for dir mode

cli/dir/dir.go

@@ -39,6 +39,8 @@ func getFlags() []cli.Flag {
 		&cli.BoolFlag{Name: "force", Value: false, Usage: "Continue even if the prechecks fail. Please only use this if you know what you are doing, it can lead to unexpected results."},
 		&cli.StringFlag{Name: "regex", Aliases: []string{"re"}, Usage: "Use regex to filter the results, by inspecting the content of the response body. When using this option be sure to set the status-codes and status-codes-blacklist options accordingly. The regex check is done after the status code checks. Only responses matching the regex will be displayed."},
 		&cli.StringFlag{Name: "regex-invert", Aliases: []string{"rei"}, Usage: "Use regex to filter the results, but inverted, by inspecting the content of the response body. When using this option be sure to set the status-codes and status-codes-blacklist options accordingly. The regex check is done after the status code checks. Only responses NOT matching the regex will be displayed."},
+		&cli.BoolFlag{Name: "stop-on-rate-limit", Value: false, Usage: "Stop the scan gracefully when an HTTP 429 (Too Many Requests) response is received"},
+		&cli.BoolFlag{Name: "retry-on-rate-limit", Value: false, Usage: "When an HTTP 429 response is received, wait for the Retry-After duration (or 5s default) then retry the request"},
 	}...)
 	return flags
 }
@@ -103,6 +105,12 @@ func run(c *cli.Context) error {
 	pluginOpts.HideLength = c.Bool("hide-length")
 	pluginOpts.DiscoverBackup = c.Bool("discover-backup")
 	pluginOpts.Force = c.Bool("force")
+	pluginOpts.StopOnRateLimit = c.Bool("stop-on-rate-limit")
+	pluginOpts.RetryOnRateLimit = c.Bool("retry-on-rate-limit")
+
+	if pluginOpts.StopOnRateLimit && pluginOpts.RetryOnRateLimit {
+		return fmt.Errorf("--stop-on-rate-limit and --retry-on-rate-limit are mutually exclusive, please set only one")
+	}
 	pluginOpts.ExcludeLength = c.String("exclude-length")
 	ret4, err := libgobuster.ParseCommaSeparatedInt(pluginOpts.ExcludeLength)
 	if err != nil {

gobusterdir/gobusterdir.go

@@ -10,15 +10,19 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"syscall"
 	"text/tabwriter"
+	"time"
 	"unicode/utf8"
 
 	"github.com/OJ/gobuster/v3/libgobuster"
 	"github.com/google/uuid"
 )
 
+const defaultRetryAfterSeconds = 5
+
 // nolint:gochecknoglobals
 var (
 	backupExtensions    = []string{"~", ".bak", ".bak2", ".old", ".1"}
@@ -304,6 +308,49 @@ func (d *GobusterDir) ProcessWord(ctx context.Context, word string, progress *li
 			}
 			return nil, err
 		}
+
+		// Handle HTTP 429 Too Many Requests
+		if statusCode == http.StatusTooManyRequests {
+			if d.options.StopOnRateLimit {
+				progress.MessageChan <- libgobuster.Message{
+					Level:   libgobuster.LevelError,
+					Message: fmt.Sprintf("Rate limit hit (HTTP 429) on %s - stopping scan as --stop-on-rate-limit is set", url.String()),
+				}
+				if progress.CancelFunc != nil {
+					progress.CancelFunc()
+				}
+				return nil, nil // nolint:nilnil
+			}
+
+			if d.options.RetryOnRateLimit {
+				waitDuration := time.Duration(defaultRetryAfterSeconds) * time.Second
+				if retryAfter := header.Get("Retry-After"); retryAfter != "" {
+					if seconds, parseErr := strconv.Atoi(retryAfter); parseErr == nil {
+						waitDuration = time.Duration(seconds) * time.Second
+					} else if retryTime, parseErr := http.ParseTime(retryAfter); parseErr == nil {
+						waitDuration = time.Until(retryTime)
+						if waitDuration < 0 {
+							waitDuration = time.Duration(defaultRetryAfterSeconds) * time.Second
+						}
+					}
+				}
+
+				progress.MessageChan <- libgobuster.Message{
+					Level:   libgobuster.LevelWarn,
+					Message: fmt.Sprintf("Rate limit hit (HTTP 429) on %s - retrying after %s", url.String(), waitDuration),
+				}
+
+				select {
+				case <-ctx.Done():
+					return nil, nil // nolint:nilnil
+				case <-time.After(waitDuration):
+				}
+				// retry this attempt (don't increment i for rate limit retries)
+				i--
+				continue
+			}
+		}
+
 		break
 	}
 
@@ -499,6 +546,18 @@ func (d *GobusterDir) GetConfigString() (string, error) {
 		}
 	}
 
+	if o.StopOnRateLimit {
+		if _, err := fmt.Fprintf(tw, "[+] Stop on rate limit:\ttrue\n"); err != nil {
+			return "", err
+		}
+	}
+
+	if o.RetryOnRateLimit {
+		if _, err := fmt.Fprintf(tw, "[+] Retry on rate limit:\ttrue\n"); err != nil {
+			return "", err
+		}
+	}
+
 	if _, err := fmt.Fprintf(tw, "[+] Timeout:\t%s\n", o.Timeout.String()); err != nil {
 		return "", err
 	}

gobusterdir/options.go

@@ -26,6 +26,8 @@ type OptionsDir struct {
 	Force                      bool
 	Regex                      *regexp.Regexp
 	RegexInvert                bool
+	StopOnRateLimit            bool
+	RetryOnRateLimit           bool
 }
 
 // NewOptions returns a new initialized OptionsDir

libgobuster/libgobuster.go

@@ -237,6 +237,9 @@ func (g *Gobuster) Run(ctx context.Context) error {
 	feederCtx, feederCancel := context.WithCancel(ctx)
 	defer feederCancel()
 
+	// Allow plugins to signal a graceful stop (e.g. on rate limiting)
+	g.Progress.CancelFunc = workerCancel
+
 	var workerGroup, feederGroup sync.WaitGroup
 	workerGroup.Add(g.Opts.Threads)
 

libgobuster/progress.go

@@ -1,6 +1,9 @@
 package libgobuster
 
-import "sync"
+import (
+	"context"
+	"sync"
+)
 
 type MessageLevel int
 
@@ -24,6 +27,8 @@ type Progress struct {
 	ResultChan            chan Result
 	ErrorChan             chan error
 	MessageChan           chan Message
+	// CancelFunc can be set by the runner to allow plugins to signal a graceful stop
+	CancelFunc context.CancelFunc
 }
 
 func NewProgress() *Progress {