diff --git a/v2/go.mod b/v2/go.mod index e40a39a4..a466229e 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -6,20 +6,21 @@ require ( github.com/Knetic/govaluate v3.0.0+incompatible github.com/blang/semver v3.5.1+incompatible github.com/d5/tengo/v2 v2.6.2 + github.com/golang/snappy v0.0.2 // indirect github.com/google/go-github/v32 v32.1.0 github.com/json-iterator/go v1.1.10 github.com/karrick/godirwalk v1.16.1 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/miekg/dns v1.1.34 + github.com/miekg/dns v1.1.35 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.4 github.com/projectdiscovery/collaborator v0.0.0-20201023080839-2aa1290ed09d github.com/projectdiscovery/gologger v1.0.1 - github.com/projectdiscovery/hmap v0.0.0-20201018163424-9cdfe6188601 + github.com/projectdiscovery/hmap v0.0.0-20201026185329-db41b5717bcb github.com/projectdiscovery/httpx v1.0.2 - github.com/projectdiscovery/rawhttp v0.0.4 + github.com/projectdiscovery/rawhttp v0.0.5-0.20201030111302-fd184be37926 github.com/projectdiscovery/retryabledns v1.0.4 github.com/projectdiscovery/retryablehttp-go v1.0.1 github.com/remeh/sizedwaitgroup v1.0.0 @@ -27,7 +28,7 @@ require ( github.com/stretchr/testify v1.5.1 go.uber.org/ratelimit v0.1.0 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 // indirect - golang.org/x/net v0.0.0-20201022231255-08b38378de70 - golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd // indirect + golang.org/x/net v0.0.0-20201029221708-28c70e62bb1d + golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 // indirect gopkg.in/yaml.v2 v2.3.0 ) diff --git a/v2/go.sum b/v2/go.sum index b2cd20e8..0a417498 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -26,6 +26,8 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= @@ -47,6 +49,8 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.34 h1:SgTzfkN+oLoIHF1bgUP+C71mzuDl3AhLApHzCCIAMWM= github.com/miekg/dns v1.1.34/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs= +github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -66,15 +70,21 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/collaborator v0.0.0-20201023080839-2aa1290ed09d h1:iHb2v6VX1Fjl9IX8UaJFLKLej+KyJJ59W1pWIRCug4Q= github.com/projectdiscovery/collaborator v0.0.0-20201023080839-2aa1290ed09d/go.mod h1:M7Csn+hQVDOLCEEFkj6dazmtgG1tIqJpbuPHlRlpYGQ= +github.com/projectdiscovery/collaborator v0.0.0-20201023192422-4a25cca69447 h1:1+NNwnPD0Rm7r4NJaX6IQwi+DiTJUc3JC6k0Zuc0/Qc= +github.com/projectdiscovery/collaborator v0.0.0-20201023192422-4a25cca69447/go.mod h1:M7Csn+hQVDOLCEEFkj6dazmtgG1tIqJpbuPHlRlpYGQ= github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/hmap v0.0.0-20201018163424-9cdfe6188601 h1:08fefqfMG8xmhexCYWC2R9XGIkGoSgVEHhBBKkdY24k= github.com/projectdiscovery/hmap v0.0.0-20201018163424-9cdfe6188601/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= +github.com/projectdiscovery/hmap v0.0.0-20201026185329-db41b5717bcb h1:y+O2ZCGICrGcSevxu2nydZtjMiwDHPf2cMgwi8PCFgQ= +github.com/projectdiscovery/hmap v0.0.0-20201026185329-db41b5717bcb/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= github.com/projectdiscovery/httpx v1.0.2 h1:g7EeRAPckZgWcHkcAH2Qzv9MkRACVRLF+T2LJcM7SCk= github.com/projectdiscovery/httpx v1.0.2/go.mod h1:OwvMc5ogx69xukKXY6kIrDP6dgOYr4VtEWyr6o573Xs= github.com/projectdiscovery/rawhttp v0.0.4 h1:O5IreNGk83d4xTD9e6SpkKbX0sHTs8K1Q33Bz4eYl2E= github.com/projectdiscovery/rawhttp v0.0.4/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= +github.com/projectdiscovery/rawhttp v0.0.5-0.20201030111302-fd184be37926 h1:aAh1EqrurioC8OeOJAzZG2gYN7MEmMNc2UWLJnQNV78= +github.com/projectdiscovery/rawhttp v0.0.5-0.20201030111302-fd184be37926/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60= github.com/projectdiscovery/retryabledns v1.0.4/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ= github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE= @@ -112,6 +122,8 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201022231255-08b38378de70 h1:Z6x4N9mAi4oF0TbHweCsH618MO6OI6UFgV0FP5n0wBY= golang.org/x/net v0.0.0-20201022231255-08b38378de70/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201029221708-28c70e62bb1d h1:dOiJ2n2cMwGLce/74I/QHMbnpk5GfY7InR8rczoMqRM= +golang.org/x/net v0.0.0-20201029221708-28c70e62bb1d/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= @@ -125,6 +137,8 @@ golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd h1:WgqgiQvkiZWz7XLhphjt2GI2GcGCTIZs9jqXMWmH+oc= golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 h1:HlFl4V6pEMziuLXyRkm5BIYq1y1GAbb02pRlWvI54OM= +golang.org/x/sys v0.0.0-20201029080932-201ba4db2418/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index f5cbe28e..d0e52234 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -174,9 +174,9 @@ func New(options *Options) (*Runner, error) { // Create the output file if asked if options.Output != "" { - output, err := bufwriter.New(options.Output) - if err != nil { - gologger.Fatalf("Could not create output file '%s': %s\n", options.Output, err) + output, errBufWriter := bufwriter.New(options.Output) + if errBufWriter != nil { + gologger.Fatalf("Could not create output file '%s': %s\n", options.Output, errBufWriter) } runner.output = output } @@ -186,10 +186,10 @@ func New(options *Options) (*Runner, error) { // create project file if requested or load existing one if options.Project { - var err error - runner.pf, err = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""}) - if err != nil { - return nil, err + var projectFileErr error + runner.pf, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""}) + if projectFileErr != nil { + return nil, projectFileErr } } diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index b768f451..355717de 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -109,10 +109,7 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) { } // Create the HTTP Client - client, err := makeHTTPClient(proxyURL, options) - if err != nil { - return nil, err - } + client := makeHTTPClient(proxyURL, options) // nolint:bodyclose // false positive there is no body to close yet client.CheckRetry = retryablehttp.HostSprayRetryPolicy() @@ -576,7 +573,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, func (e *HTTPExecuter) Close() {} // makeHTTPClient creates a http client -func makeHTTPClient(proxyURL *url.URL, options *HTTPOptions) (*retryablehttp.Client, error) { +func makeHTTPClient(proxyURL *url.URL, options *HTTPOptions) *retryablehttp.Client { // Multiple Host retryablehttpOptions := retryablehttp.DefaultOptionsSpraying disableKeepAlives := true @@ -639,7 +636,7 @@ func makeHTTPClient(proxyURL *url.URL, options *HTTPOptions) (*retryablehttp.Cli Transport: transport, Timeout: time.Duration(options.Timeout) * time.Second, CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects), - }, retryablehttpOptions), nil + }, retryablehttpOptions) } type checkRedirectFunc func(_ *http.Request, requests []*http.Request) error diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index f9d0238e..b90a0836 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -278,7 +278,6 @@ func (r *BulkHTTPRequest) fillRequest(req *http.Request, values map[string]inter // In case of multiple threads the underlying connection should remain open to allow reuse if r.Threads <= 0 && req.Header.Get("Connection") == "" { - setHeader(req, "Connection", "close") req.Close = true } diff --git a/v2/pkg/requests/dump.go b/v2/pkg/requests/dump.go index d98dae27..25651513 100644 --- a/v2/pkg/requests/dump.go +++ b/v2/pkg/requests/dump.go @@ -1,6 +1,7 @@ package requests import ( + "bytes" "io/ioutil" "net/http/httputil" "strings" @@ -10,6 +11,9 @@ import ( func Dump(req *HTTPRequest, reqURL string) ([]byte, error) { if req.Request != nil { + // Create a copy on the fly of the request body - ignore errors + bodyBytes, _ := req.Request.BodyBytes() + req.Request.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes)) return httputil.DumpRequest(req.Request.Request, true) }