diff --git a/v2/go.mod b/v2/go.mod index 14638dfe..8dab814d 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -14,13 +14,13 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/gologger v1.0.1 github.com/projectdiscovery/httpx v1.0.2 - github.com/projectdiscovery/rawhttp v0.0.3 + github.com/projectdiscovery/rawhttp v0.0.4 github.com/projectdiscovery/retryabledns v1.0.4 github.com/projectdiscovery/retryablehttp-go v1.0.1 github.com/remeh/sizedwaitgroup v1.0.0 github.com/spaolacci/murmur3 v1.1.0 github.com/vbauerster/mpb/v5 v5.3.0 go.uber.org/ratelimit v0.1.0 - golang.org/x/net v0.0.0-20201010224723-4f7140c49acb + golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0 gopkg.in/yaml.v2 v2.3.0 ) diff --git a/v2/go.sum b/v2/go.sum index 20c00ede..d64f03e2 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -51,8 +51,8 @@ github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs2 github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/httpx v1.0.2 h1:g7EeRAPckZgWcHkcAH2Qzv9MkRACVRLF+T2LJcM7SCk= github.com/projectdiscovery/httpx v1.0.2/go.mod h1:OwvMc5ogx69xukKXY6kIrDP6dgOYr4VtEWyr6o573Xs= -github.com/projectdiscovery/rawhttp v0.0.3 h1:UCNHNnRDHixtPd75kUOWi8QtIlxFnkSa7ugrKUB5Eto= -github.com/projectdiscovery/rawhttp v0.0.3/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= +github.com/projectdiscovery/rawhttp v0.0.4 h1:O5IreNGk83d4xTD9e6SpkKbX0sHTs8K1Q33Bz4eYl2E= +github.com/projectdiscovery/rawhttp v0.0.4/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60= github.com/projectdiscovery/retryabledns v1.0.4/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ= github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE= @@ -83,8 +83,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20201010224723-4f7140c49acb h1:mUVeFHoDKis5nxCAzoAi7E8Ghb86EXh/RK6wtvJIqRY= -golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0 h1:5kGOVHlq0euqwzgTC9Vu15p6fV1Wi0ArVi8da2urnVg= +golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index abff342d..d2c0e299 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -346,6 +346,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, options := e.rawHTTPClient.Options options.AutomaticContentLength = request.AutomaticContentLengthHeader options.AutomaticHostHeader = request.AutomaticHostHeader + options.FollowRedirects = request.FollowRedirects resp, err = e.rawHTTPClient.DoRawWithOptions(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)), options) if err != nil { if resp != nil { diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index 1111cae0..a87537b3 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -224,7 +224,15 @@ func (r *BulkHTTPRequest) handleRawWithPaylods(ctx context.Context, raw, baseURL // rawhttp if r.Unsafe { - return &HTTPRequest{RawRequest: rawRequest, Meta: genValues, AutomaticHostHeader: !r.DisableAutoHostname, AutomaticContentLengthHeader: !r.DisableAutoContentLength, Unsafe: true}, nil + unsafeReq := &HTTPRequest{ + RawRequest: rawRequest, + Meta: genValues, + AutomaticHostHeader: !r.DisableAutoHostname, + AutomaticContentLengthHeader: !r.DisableAutoContentLength, + Unsafe: true, + FollowRedirects: r.Redirects, + } + return unsafeReq, nil } // retryablehttp @@ -289,6 +297,7 @@ type HTTPRequest struct { AutomaticHostHeader bool AutomaticContentLengthHeader bool AutomaticConnectionHeader bool + FollowRedirects bool Rawclient *rawhttp.Client Httpclient *retryablehttp.Client PipelineClient *rawhttp.PipelineClient @@ -382,7 +391,7 @@ func (r *BulkHTTPRequest) parseRawRequest(request, baseURL string) (*RawRequest, // Handle case with the full http url in path. In that case, // ignore any host header that we encounter and use the path as request URL - if strings.HasPrefix(parts[1], "http") { + if !r.Unsafe && strings.HasPrefix(parts[1], "http") { parsed, parseErr := url.Parse(parts[1]) if parseErr != nil { return nil, fmt.Errorf("could not parse request URL: %s", parseErr)