diff --git a/go.mod b/go.mod index af4870cd..e832c99e 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/miekg/dns v1.1.29 // indirect github.com/projectdiscovery/gologger v1.0.0 github.com/projectdiscovery/massdns v0.0.0-20200331204010-f72d136ae361 + github.com/projectdiscovery/retryablehttp-go v0.0.0-20200404113336-78da8dcb5040 github.com/projectdiscovery/shuffledns v1.0.2 // indirect github.com/rs/xid v1.2.1 github.com/stretchr/testify v1.5.1 diff --git a/go.sum b/go.sum index e5736875..1b27b213 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEv github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/massdns v0.0.0-20200331204010-f72d136ae361 h1:HXHngEMCoJGgEUigUwX5L2PD1Fpj2EcQMplXgPwpkD4= github.com/projectdiscovery/massdns v0.0.0-20200331204010-f72d136ae361/go.mod h1:F4246OEiiR3iUPQDt0BRUdEZdlrQVyM+I3Wl3AcFaUg= +github.com/projectdiscovery/retryablehttp-go v0.0.0-20200404113336-78da8dcb5040 h1:vQQ/wys6mfOVVRyMw+W9lS52J6Iiy7MZp3DXFI94UY0= +github.com/projectdiscovery/retryablehttp-go v0.0.0-20200404113336-78da8dcb5040/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/projectdiscovery/shuffledns v1.0.2 h1:0gvWe6eHNaUr1xkJCRaYVi33dcFVU/bSaOmKp16mTis= github.com/projectdiscovery/shuffledns v1.0.2/go.mod h1:sp+rOgvOckLYxyzAM8WtxQKkuPyHNcG/xLWkKwLm/Vo= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 9dd4fb7a..355d65d4 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -2,6 +2,7 @@ package runner import ( "bufio" + "crypto/tls" "fmt" "io" "io/ioutil" @@ -10,15 +11,17 @@ import ( "path/filepath" "strings" "sync" + "time" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/pkg/matchers" "github.com/projectdiscovery/nuclei/pkg/templates" + retryablehttp "github.com/projectdiscovery/retryablehttp-go" ) // Runner is a client for running the enumeration process. type Runner struct { - client *http.Client + client *retryablehttp.Client options *Options } @@ -27,6 +30,26 @@ func New(options *Options) (*Runner, error) { runner := &Runner{ options: options, } + + // Create the HTTP Client + client := retryablehttp.NewWithHTTPClient(&http.Client{ + Transport: &http.Transport{ + MaxIdleConnsPerHost: -1, + TLSClientConfig: &tls.Config{ + Renegotiation: tls.RenegotiateOnceAsClient, + InsecureSkipVerify: true, + }, + DisableKeepAlives: true, + }, + Timeout: time.Duration(options.Timeout) * time.Second, + CheckRedirect: func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + }, + }, retryablehttp.DefaultOptionsSpraying) + client.Backoff = retryablehttp.FullJitterBackoff() + client.CheckRetry = retryablehttp.HostSprayRetryPolicy() + + runner.client = client return runner, nil } diff --git a/pkg/requests/requests.go b/pkg/requests/requests.go index 17a39577..1107528f 100644 --- a/pkg/requests/requests.go +++ b/pkg/requests/requests.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/projectdiscovery/nuclei/pkg/matchers" + retryablehttp "github.com/projectdiscovery/retryablehttp-go" "github.com/valyala/fasttemplate" ) @@ -26,14 +27,14 @@ type Request struct { } // MakeRequest creates a *http.Request from a request template -func (r *Request) MakeRequest(baseURL string) ([]*http.Request, error) { +func (r *Request) MakeRequest(baseURL string) ([]*retryablehttp.Request, error) { parsed, err := url.Parse(baseURL) if err != nil { return nil, err } hostname := parsed.Hostname() - requests := make([]*http.Request, 0, len(r.Path)) + requests := make([]*retryablehttp.Request, 0, len(r.Path)) for _, path := range r.Path { // Replace the dynamic variables in the URL if any t := fasttemplate.New(path, "{{", "}}") @@ -67,7 +68,12 @@ func (r *Request) MakeRequest(baseURL string) ([]*http.Request, error) { req.Header.Set("Connection", "close") req.Close = true - requests = append(requests, req) + request, err := retryablehttp.FromRequest(req) + if err != nil { + return nil, err + } + + requests = append(requests, request) } return requests, nil