Merge pull request #75 from edoardottt/devel

Devel

Author: edoardottt

.golangci.yml

@@ -51,4 +51,6 @@ linters-settings:
     # Values always ignored: "1", "1.0", "0" and "0.0"
     # Default: []
     ignored-numbers:
-      - '2'
+      - '2'
+    ignored-files:
+      - 'utils/http.go'

README.md

@@ -128,6 +128,8 @@ usage: scilla subcommand { options }
                    [-plain Print only results]
                    [-db -no-check Don't check status codes for subdomains]
                    [-db -vt Use VirusTotal as subdomains source]
+                   [-ua Set the User Agent]
+                   [-rua Generate a random user agent for each request]
                    -target <target (URL)> REQUIRED
        - dir [-w wordlist]
              [-oj JSON output file]
@@ -137,6 +139,8 @@ usage: scilla subcommand { options }
              [-c use also a web crawler]
              [-plain Print only results]
              [-nr No follow redirects]
+             [-ua Set the User Agent]
+             [-rua Generate a random user agent for each request]
              -target <target (URL)> REQUIRED
        - report [-p <start-end> or ports divided by comma]
                 [-ws subdomains wordlist]
@@ -152,6 +156,8 @@ usage: scilla subcommand { options }
                 [-common scan common ports]
                 [-nr No follow redirects]
                 [-db -vt Use VirusTotal as subdomains source]
+                [-ua Set the User Agent]
+                [-rua Generate a random user agent for each request]
                 -target <target (URL/IP)> REQUIRED
        - help
        - examples
@@ -183,6 +189,8 @@ Examples 💡
     - `scilla subdomain -plain -target target.domain`
     - `scilla subdomain -db -no-check -target target.domain`
     - `scilla subdomain -db -vt -target target.domain`
+    - `scilla subdomain -ua "CustomUA" -target target.domain`
+    - `scilla subdomain -rua -target target.domain`
 
 - Directories enumeration:
 
@@ -196,6 +204,8 @@ Examples 💡
     - `scilla dir -c -target target.domain`
     - `scilla dir -plain -target target.domain`
     - `scilla dir -nr -target target.domain`
+    - `scilla dir -ua "CustomUA" -target target.domain`
+    - `scilla dir -rua -target target.domain`
 
 - Ports enumeration:
 
@@ -234,6 +244,8 @@ Examples 💡
     - Specifying common ports `scilla report -common -target target.domain`
     - No follow redirects `scilla report -nr -target target.domain`
     - Use VirusTotal as subdomains source `scilla report -db -vt -target target.domain`
+    - Set the User Agent `scilla report -ua "CustomUA" -target target.domain`
+    - Generate a random user agent for each request `scilla report -rua -target target.domain`
 
 Changelog 📌
 -------

crawler/colly.go

@@ -38,6 +38,7 @@ import (
 	"github.com/edoardottt/scilla/output"
 	"github.com/edoardottt/scilla/utils"
 	"github.com/gocolly/colly"
+	"github.com/gocolly/colly/extensions"
 )
 
 const (
@@ -49,7 +50,7 @@ const (
 // - only http, https or ftp protocols allowed.
 func SpawnCrawler(target string, scheme string, ignore []string, dirs map[string]output.Asset,
 	subs map[string]output.Asset, outputFileJSON, outputFileHTML, outputFileTXT string,
-	mutex *sync.Mutex, what string, plain bool) {
+	mutex *sync.Mutex, what string, plain bool, ua string, rua bool) {
 	ignoreBool := len(ignore) != 0
 	//nolint:staticcheck // SA4006 ignore this!
 	collector := colly.NewCollector()
@@ -71,6 +72,18 @@ func SpawnCrawler(target string, scheme string, ignore []string, dirs map[string
 	collector.IgnoreRobotsTxt = true
 	collector.AllowURLRevisit = false
 
+	if ua != "Go http/Client" {
+		collector.UserAgent = ua
+	} else {
+		// Avoid using the default colly user agent
+		collector.UserAgent = utils.GenerateRandomUserAgent()
+	}
+
+	// Use a Random User Agent for each request if needed
+	if rua {
+		extensions.RandomUserAgent(collector)
+	}
+
 	// Find and visit all links
 	collector.OnHTML("a[href]", func(element *colly.HTMLElement) {
 		link := element.Attr("href")

enumeration/dirs.go

@@ -40,7 +40,7 @@ import (
 // AsyncDir performs concurrent requests to the specified
 // urls and prints the results.
 func AsyncDir(urls []string, ignore []string, outputFileJSON, outputFileHTML, outputFileTXT string,
-	dirs map[string]output.Asset, mutex *sync.Mutex, plain bool, redirect bool) {
+	dirs map[string]output.Asset, mutex *sync.Mutex, plain bool, redirect bool, ua string, rua bool) {
 	ignoreBool := len(ignore) != 0
 	total := len(urls)
 	client := http.Client{}
@@ -86,7 +86,23 @@ func AsyncDir(urls []string, ignore []string, outputFileJSON, outputFileHTML, ou
 			defer waitgroup.Done()
 			defer func() { <-limiter }()
 
-			resp, err := client.Get(domain)
+			req, err := http.NewRequest("GET", domain, nil)
+			if err != nil {
+				return
+			}
+
+			if ua != "Go http/Client" {
+				req.Header.Set("User-Agent", ua)
+			}
+
+			if rua {
+				req.Header.Set("User-Agent", utils.GenerateRandomUserAgent())
+			}
+
+			resp, err := client.Do(req)
+			if err != nil {
+				return
+			}
 
 			count++
 

enumeration/subs.go

@@ -40,7 +40,7 @@ import (
 // AsyncGet performs concurrent requests to the specified
 // urls and prints the results.
 func AsyncGet(protocol string, urls []string, ignore []string, outputFileJSON, outputFileHTML, outputFileTXT string,
-	subs map[string]output.Asset, mutex *sync.Mutex, plain bool) {
+	subs map[string]output.Asset, mutex *sync.Mutex, plain bool, ua string, rua bool) {
 	ignoreBool := len(ignore) != 0
 
 	var count int
@@ -77,7 +77,23 @@ func AsyncGet(protocol string, urls []string, ignore []string, outputFileJSON, o
 			defer waitgroup.Done()
 			defer func() { <-limiter }()
 
-			resp, err := client.Get(protocol + "://" + domain)
+			req, err := http.NewRequest("GET", protocol+"://"+domain, nil)
+			if err != nil {
+				return
+			}
+
+			if ua != "Go http/Client" {
+				req.Header.Set("User-Agent", ua)
+			}
+
+			if rua {
+				req.Header.Set("User-Agent", utils.GenerateRandomUserAgent())
+			}
+
+			resp, err := client.Do(req)
+			if err != nil {
+				return
+			}
 
 			count++
 

input/check.go

@@ -42,8 +42,8 @@ func ReportSubcommandCheckFlags(reportCommand flag.FlagSet, reportTargetPtr *str
 	reportPortsPtr *string, reportCommonPtr *bool, reportVirusTotalPtr *bool, reportSubdomainDBPtr *bool,
 	startPort int, endPort int, reportIgnoreDirPtr *string,
 	reportIgnoreSubPtr *string, reportTimeoutPort *int,
-	reportOutputJSON *string, reportOutputHTML *string, reportOutputTXT *string) (int, int,
-	[]int, bool, []string, []string) {
+	reportOutputJSON, reportOutputHTML, reportOutputTXT, reportUserAgentPtr *string,
+	reportRandomUserAgentPtr *bool) (int, int, []int, bool, []string, []string) {
 	// Required Flags
 	if *reportTargetPtr == "" {
 		reportCommand.PrintDefaults()
@@ -142,6 +142,11 @@ func ReportSubcommandCheckFlags(reportCommand flag.FlagSet, reportTargetPtr *str
 		os.Exit(1)
 	}
 
+	if *reportUserAgentPtr != DefaultUserAgent && *reportRandomUserAgentPtr {
+		fmt.Println("You cannot specify both ua and rua.")
+		os.Exit(1)
+	}
+
 	return startPort, endPort, portsArray, portArrayBool, reportIgnoreDir, reportIgnoreSub
 }
 
@@ -189,7 +194,8 @@ func DNSSubcommandCheckFlags(dnsCommand flag.FlagSet, dnsTargetPtr, dnsOutputJSO
 func SubdomainSubcommandCheckFlags(subdomainCommand flag.FlagSet, subdomainTargetPtr *string,
 	subdomainNoCheckPtr *bool, subdomainDBPtr *bool, subdomainWordlistPtr *string,
 	subdomainIgnorePtr *string, subdomainCrawlerPtr *bool, subdomainVirusTotalPtr *bool,
-	subdomainOutputJSON, subdomainOutputHTML, subdomainOutputTXT *string) []string {
+	subdomainOutputJSON, subdomainOutputHTML, subdomainOutputTXT, subdomainUserAgentPtr *string,
+	subdomainRandomUserAgentPtr *bool) []string {
 	// Required Flags
 	if *subdomainTargetPtr == "" {
 		subdomainCommand.PrintDefaults()
@@ -258,6 +264,11 @@ func SubdomainSubcommandCheckFlags(subdomainCommand flag.FlagSet, subdomainTarge
 		subdomainIgnore = utils.CheckIgnore(toBeIgnored)
 	}
 
+	if *subdomainUserAgentPtr != DefaultUserAgent && *subdomainRandomUserAgentPtr {
+		fmt.Println("You cannot specify both ua and rua.")
+		os.Exit(1)
+	}
+
 	return subdomainIgnore
 }
 
@@ -352,7 +363,8 @@ func PortSubcommandCheckFlags(portCommand flag.FlagSet, portTargetPtr *string, p
 // DirSubcommandCheckFlags performs all the necessary checks on the flags
 // for the dir subcommand.
 func DirSubcommandCheckFlags(dirCommand flag.FlagSet, dirTargetPtr *string,
-	dirIgnorePtr *string, dirOutputJSON, dirOutputHTML, dirOutputTXT *string) []string {
+	dirIgnorePtr *string, dirOutputJSON, dirOutputHTML, dirOutputTXT, dirUserAgentPtr *string,
+	dirRandomUserAgentPtr *bool) []string {
 	// Required Flags
 	if *dirTargetPtr == "" {
 		dirCommand.PrintDefaults()
@@ -394,5 +406,10 @@ func DirSubcommandCheckFlags(dirCommand flag.FlagSet, dirTargetPtr *string,
 		dirIgnore = utils.CheckIgnore(toBeIgnored)
 	}
 
+	if *dirUserAgentPtr != DefaultUserAgent && *dirRandomUserAgentPtr {
+		fmt.Println("You cannot specify both ua and rua.")
+		os.Exit(1)
+	}
+
 	return dirIgnore
 }