daffainfo
/
subfinder
mirror of https://github.com/daffainfo/subfinder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #449 from taythebot/master 

Add Rate Limit and User Agent randomization for HTTP requests
dnsrepo-source
Sandeep Singh 2021-08-16 17:37:13 +05:30 committed by GitHub
parent cc69c17043 6c7c2d9ac8
commit 08671cf155
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 36 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -76,6 +76,7 @@ This will display help for the tool. Here are all the switches it supports.
| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 |
| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 |
| -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 |
| -rate-limit | Maximum number of HTTP requests to send per second | subfinder -rate-limit 10 |
| -v | Show Verbose output | subfinder -v |
| -version | Show current program version | subfinder -version |

2
v2/go.mod
View File

@ -3,6 +3,7 @@ module github.com/projectdiscovery/subfinder/v2
go 1.16
require (
github.com/corpix/uarand v0.1.1
github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd
github.com/json-iterator/go v1.1.10
github.com/lib/pq v1.10.0
@ -14,5 +15,6 @@ require (
github.com/spyse-com/go-spyse v1.2.1
github.com/stretchr/testify v1.7.0
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
go.uber.org/ratelimit v0.2.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)

11
v2/go.sum
View File

@ -1,8 +1,15 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@ -51,6 +58,7 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -59,6 +67,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@ -101,7 +110,9 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

4
v2/pkg/passive/passive.go
View File

@ -11,11 +11,11 @@ import (
)
// EnumerateSubdomains enumerates all the subdomains for a given domain
func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result {
func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result {
results := make(chan subscraping.Result)
go func() {
session, err := subscraping.NewSession(domain, keys, proxy, timeout)
session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout)
if err != nil {
results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)}
}

2
v2/pkg/runner/enumerate.go
View File

@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu
// Run the passive subdomain enumeration
now := time.Now()
passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute)
passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.RateLimit, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute)
wg := &sync.WaitGroup{}
wg.Add(1)

2
v2/pkg/runner/options.go
View File

@ -40,6 +40,7 @@ type Options struct {
ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration
ConfigFile string // ConfigFile contains the location of the config file
Proxy string // HTTP proxy
RateLimit int // Maximum number of HTTP requests to send per second
YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file
}
@ -77,6 +78,7 @@ func ParseOptions() *Options {
flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output")
flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc")
flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use")
flag.IntVar(&options.RateLimit, "rate-limit", 0, "Maximum number of HTTP requests to send per second")
flag.BoolVar(&options.Version, "version", false, "Show version of subfinder")
flag.Parse()

15
v2/pkg/subscraping/agent.go
View File

@ -10,11 +10,13 @@ import (
"net/url"
"time"
"github.com/corpix/uarand"
"github.com/projectdiscovery/gologger"
"go.uber.org/ratelimit"
)
// NewSession creates a new session object for a domain
func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) {
func NewSession(domain string, keys *Keys, proxy string, rateLimit, timeout int) (*Session, error) {
Transport := &http.Transport{
MaxIdleConns: 100,
MaxIdleConnsPerHost: 100,
@ -44,6 +46,13 @@ func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session,
Keys: keys,
}
// Initiate rate limit instance
if rateLimit > 0 {
session.RateLimiter = ratelimit.New(rateLimit)
} else {
session.RateLimiter = ratelimit.NewUnlimited()
}
// Create a new extractor object for the current domain
extractor, err := NewSubdomainExtractor(domain)
session.Extractor = extractor
@ -78,7 +87,7 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s
return nil, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36")
req.Header.Set("User-Agent", uarand.GetRandom())
req.Header.Set("Accept", "*/*")
req.Header.Set("Accept-Language", "en")
req.Header.Set("Connection", "close")
@ -95,6 +104,8 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s
req.Header.Set(key, value)
}
s.RateLimiter.Take()
return httpRequestWrapper(s.Client, req)
}

4
v2/pkg/subscraping/types.go
View File

@ -4,6 +4,8 @@ import (
"context"
"net/http"
"regexp"
"go.uber.org/ratelimit"
)
// BasicAuth request's Authorization header
@ -31,6 +33,8 @@ type Session struct {
Keys *Keys
// Client is the current http client
Client *http.Client
// Rate limit instance
RateLimiter ratelimit.Limiter
}
// Keys contains the current API Keys we have in store