From e5d9f1a58311c6cc43152d3ec5d5431a5155913d Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 11 Aug 2021 21:25:07 +0900 Subject: [PATCH 1/9] User Agent Randomization HTTP User-Agent header randomization. Adds -unsafe flag to send requests with default User-Agent header. --- v2/go.mod | 1 + v2/go.sum | 8 ++++++++ v2/pkg/passive/passive.go | 4 ++-- v2/pkg/runner/enumerate.go | 2 +- v2/pkg/runner/options.go | 2 ++ v2/pkg/subscraping/agent.go | 12 ++++++++++-- v2/pkg/subscraping/types.go | 2 ++ 7 files changed, 26 insertions(+), 5 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 543cf9e..7bd474d 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -3,6 +3,7 @@ module github.com/projectdiscovery/subfinder/v2 go 1.16 require ( + github.com/corpix/uarand v0.1.1 github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd github.com/json-iterator/go v1.1.10 github.com/lib/pq v1.10.0 diff --git a/v2/go.sum b/v2/go.sum index 0c27ad7..e7a7da7 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1,8 +1,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= +github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= +github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -51,12 +57,14 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index ace9846..e2aedae 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, unsafe bool, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, unsafe, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 84cbb03..2e61c83 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.UnSafe, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 6525c2b..c7d9d94 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,6 +40,7 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy + UnSafe bool // Send HTTP request without User-Agent header randomization YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } @@ -77,6 +78,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") + flag.BoolVar(&options.UnSafe, "unsafe", false,"Send HTTP request without User-Agent header randomization") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index a039dfc..e82a596 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -10,11 +10,12 @@ import ( "net/url" "time" + "github.com/corpix/uarand" "github.com/projectdiscovery/gologger" ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, unsafe bool, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -42,6 +43,7 @@ func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, session := &Session{ Client: client, Keys: keys, + UnSafe: unsafe, } // Create a new extractor object for the current domain @@ -78,7 +80,13 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s return nil, err } - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") + // Unsafe requests do not use user-agent randomization + if s.UnSafe { + req.Header.Set("User-Agent", "subfinder - Open-source Project (github.com/projectdiscovery/subfinder)") + } else { + req.Header.Set("User-Agent", uarand.GetRandom()) + } + req.Header.Set("Accept", "*/*") req.Header.Set("Accept-Language", "en") req.Header.Set("Connection", "close") diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index e3211be..9df42ff 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -31,6 +31,8 @@ type Session struct { Keys *Keys // Client is the current http client Client *http.Client + // Perform unsafe HTTP requests without user agent randomization + UnSafe bool } // Keys contains the current API Keys we have in store From 5bc901c47d0a22b6c1d5a2ea06ca65ad4bfdccfd Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 11 Aug 2021 21:30:15 +0900 Subject: [PATCH 2/9] Add unsafe flag to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fe06180..c907528 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ This will display help for the tool. Here are all the switches it supports. | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | | -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 | +| -unsafe | Send HTTP request without User-Agent header randomization | subfinder -unsafe | | -v | Show Verbose output | subfinder -v | | -version | Show current program version | subfinder -version | From f35adaf14356b6d6f4efe7b1241105c4829bc114 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 11 Aug 2021 21:34:20 +0900 Subject: [PATCH 3/9] Fix lint issue --- v2/pkg/runner/options.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index c7d9d94..ce9c317 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,7 +40,7 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy - UnSafe bool // Send HTTP request without User-Agent header randomization + UnSafe bool // Send HTTP request without User-Agent header randomization YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } @@ -78,7 +78,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") - flag.BoolVar(&options.UnSafe, "unsafe", false,"Send HTTP request without User-Agent header randomization") + flag.BoolVar(&options.UnSafe, "unsafe", false, "Send HTTP request without User-Agent header randomization") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() From 095d4295f230b4cbb26ee0bed47e63c2b0d36e67 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 11 Aug 2021 21:42:52 +0900 Subject: [PATCH 4/9] Remove unsafe flag HTTP User-Agent should be randomized by default since subfinder uses third party apis --- README.md | 1 - v2/pkg/passive/passive.go | 4 ++-- v2/pkg/runner/enumerate.go | 2 +- v2/pkg/runner/options.go | 2 -- v2/pkg/subscraping/agent.go | 11 ++--------- v2/pkg/subscraping/types.go | 2 -- 6 files changed, 5 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index c907528..fe06180 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,6 @@ This will display help for the tool. Here are all the switches it supports. | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | | -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 | -| -unsafe | Send HTTP request without User-Agent header randomization | subfinder -unsafe | | -v | Show Verbose output | subfinder -v | | -version | Show current program version | subfinder -version | diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index e2aedae..ace9846 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, unsafe bool, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, unsafe, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 2e61c83..84cbb03 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.UnSafe, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index ce9c317..6525c2b 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,7 +40,6 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy - UnSafe bool // Send HTTP request without User-Agent header randomization YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } @@ -78,7 +77,6 @@ func ParseOptions() *Options { flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") - flag.BoolVar(&options.UnSafe, "unsafe", false, "Send HTTP request without User-Agent header randomization") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index e82a596..6b89337 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -15,7 +15,7 @@ import ( ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, unsafe bool, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -43,7 +43,6 @@ func NewSession(domain string, keys *Keys, proxy string, unsafe bool, timeout in session := &Session{ Client: client, Keys: keys, - UnSafe: unsafe, } // Create a new extractor object for the current domain @@ -80,13 +79,7 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s return nil, err } - // Unsafe requests do not use user-agent randomization - if s.UnSafe { - req.Header.Set("User-Agent", "subfinder - Open-source Project (github.com/projectdiscovery/subfinder)") - } else { - req.Header.Set("User-Agent", uarand.GetRandom()) - } - + req.Header.Set("User-Agent", uarand.GetRandom()) req.Header.Set("Accept", "*/*") req.Header.Set("Accept-Language", "en") req.Header.Set("Connection", "close") diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 9df42ff..e3211be 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -31,8 +31,6 @@ type Session struct { Keys *Keys // Client is the current http client Client *http.Client - // Perform unsafe HTTP requests without user agent randomization - UnSafe bool } // Keys contains the current API Keys we have in store From ec79e8053a8f6c0a018bad8c409ef1897b105efe Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Thu, 12 Aug 2021 08:29:58 +0900 Subject: [PATCH 5/9] Add rate limit feature --- v2/go.mod | 1 + v2/go.sum | 3 +++ v2/pkg/passive/passive.go | 4 ++-- v2/pkg/runner/enumerate.go | 2 +- v2/pkg/runner/options.go | 2 ++ v2/pkg/subscraping/agent.go | 12 +++++++++++- v2/pkg/subscraping/types.go | 4 ++++ 7 files changed, 24 insertions(+), 4 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 7bd474d..3e7c5c9 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -14,5 +14,6 @@ require ( github.com/rs/xid v1.3.0 github.com/stretchr/testify v1.7.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 + go.uber.org/ratelimit v0.2.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/v2/go.sum b/v2/go.sum index e7a7da7..1b4b092 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -3,6 +3,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -105,7 +106,9 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index ace9846..91e44e9 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit int, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 84cbb03..5df3fb6 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.RateLimit, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 6525c2b..e672d0b 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,6 +40,7 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy + RateLimit int // Maximum number of HTTP requests to send per second YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } @@ -77,6 +78,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") + flag.IntVar(&options.RateLimit, "rate-limit", 0, "Maximum number of HTTP requests to send per second") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index 6b89337..3fafe12 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -12,10 +12,11 @@ import ( "github.com/corpix/uarand" "github.com/projectdiscovery/gologger" + "go.uber.org/ratelimit" ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, rateLimit int, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -45,6 +46,13 @@ func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, Keys: keys, } + // Initiate rate limit instance + if rateLimit > 0 { + session.RateLimiter = ratelimit.New(rateLimit) + } else { + session.RateLimiter = ratelimit.NewUnlimited() + } + // Create a new extractor object for the current domain extractor, err := NewSubdomainExtractor(domain) session.Extractor = extractor @@ -96,6 +104,8 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s req.Header.Set(key, value) } + s.RateLimiter.Take() + return httpRequestWrapper(s.Client, req) } diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index e3211be..f49187a 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -4,6 +4,8 @@ import ( "context" "net/http" "regexp" + + "go.uber.org/ratelimit" ) // BasicAuth request's Authorization header @@ -31,6 +33,8 @@ type Session struct { Keys *Keys // Client is the current http client Client *http.Client + // Rate limit instance + RateLimiter ratelimit.Limiter } // Keys contains the current API Keys we have in store From f9381ceec9352301276179b8e656e75eee9ed66d Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Thu, 12 Aug 2021 08:33:18 +0900 Subject: [PATCH 6/9] Add rate-limit flag to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fe06180..7f55e62 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ This will display help for the tool. Here are all the switches it supports. | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | | -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 | +| -rate-limit | Maximum number of HTTP requests to send per second | subfinder -rate-limit 10 | | -v | Show Verbose output | subfinder -v | | -version | Show current program version | subfinder -version | From cb548cad51329a19e97ad49987512d80393bd6ed Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Thu, 12 Aug 2021 08:36:26 +0900 Subject: [PATCH 7/9] Update options.go --- v2/pkg/runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index e672d0b..9413538 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,7 +40,7 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy - RateLimit int // Maximum number of HTTP requests to send per second + RateLimit int // Maximum number of HTTP requests to send per second YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } From a1e27912719c01adb13e674706cdc3b87997a507 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Fri, 13 Aug 2021 07:39:05 +0900 Subject: [PATCH 8/9] Change variable rateLimit to rl --- v2/pkg/passive/passive.go | 4 ++-- v2/pkg/subscraping/agent.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index 91e44e9..d925701 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit int, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rl int, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, rl, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index 3fafe12..40c5a67 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -16,7 +16,7 @@ import ( ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, rateLimit int, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, rl int, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -47,8 +47,8 @@ func NewSession(domain string, keys *Keys, proxy string, rateLimit int, timeout } // Initiate rate limit instance - if rateLimit > 0 { - session.RateLimiter = ratelimit.New(rateLimit) + if rl > 0 { + session.RateLimiter = ratelimit.New(rl) } else { session.RateLimiter = ratelimit.NewUnlimited() } From 0aac1eb68567ab68dfefc75e511ba67718473339 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Fri, 13 Aug 2021 07:41:11 +0900 Subject: [PATCH 9/9] Fix lint --- v2/pkg/passive/passive.go | 4 ++-- v2/pkg/subscraping/agent.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index d925701..0ed98d4 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rl int, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, rl, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index 40c5a67..6797542 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -16,7 +16,7 @@ import ( ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, rl int, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, rateLimit, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -47,8 +47,8 @@ func NewSession(domain string, keys *Keys, proxy string, rl int, timeout int) (* } // Initiate rate limit instance - if rl > 0 { - session.RateLimiter = ratelimit.New(rl) + if rateLimit > 0 { + session.RateLimiter = ratelimit.New(rateLimit) } else { session.RateLimiter = ratelimit.NewUnlimited() }