From d963fc08406ca6ac0809c3405df7b12bcf4d7d55 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 1 Feb 2021 16:21:49 +0530 Subject: [PATCH] Fixed rawhttp header formatting issues --- v2/pkg/protocols/common/clusterer/executer.go | 9 +- v2/pkg/protocols/common/executer/executer.go | 8 +- .../protocols/common/interactsh/interactsh.go | 127 ++++++++++++++++++ v2/pkg/protocols/http/http.go | 9 +- v2/pkg/protocols/http/raw/raw.go | 2 +- v2/pkg/protocols/http/request.go | 23 +--- 6 files changed, 148 insertions(+), 30 deletions(-) create mode 100644 v2/pkg/protocols/common/interactsh/interactsh.go diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/protocols/common/clusterer/executer.go index 0d78b00e..cef218a3 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/protocols/common/clusterer/executer.go @@ -76,16 +76,13 @@ func (e *Executer) Execute(input string) (bool, error) { } } }) - if err != nil { - return results, err - } - return results, nil + return results, err } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { dynamicValues := make(map[string]interface{}) - _ = e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) { + err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract) if matched && result != nil { @@ -97,5 +94,5 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve } } }) - return nil + return err } diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index ff2ee0f1..50e0c3b7 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -3,6 +3,7 @@ package executer import ( "strings" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" ) @@ -69,7 +70,7 @@ func (e *Executer) Execute(input string) (bool, error) { } }) if err != nil { - continue + gologger.Warning().Msgf("Could not execute request for %s: %s\n", e.options.TemplateID, err) } } return results, nil @@ -81,7 +82,7 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve previous := make(map[string]interface{}) for _, req := range e.requests { - _ = req.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { + err := req.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { ID := req.GetID() if ID != "" { builder := &strings.Builder{} @@ -98,6 +99,9 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve } callback(event) }) + if err != nil { + gologger.Warning().Msgf("Could not execute request for %s: %s\n", e.options.TemplateID, err) + } } return nil } diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go new file mode 100644 index 00000000..65991fcb --- /dev/null +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -0,0 +1,127 @@ +package interactsh + +import ( + "net/url" + "strings" + "time" + + "github.com/karlseguin/ccache" + "github.com/pkg/errors" + "github.com/projectdiscovery/interactsh/pkg/client" + "github.com/projectdiscovery/interactsh/pkg/server" + "github.com/projectdiscovery/nuclei/v2/internal/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/output" +) + +// Client is a wrapped client for interactsh server. +type Client struct { + // interactsh is a client for interactsh server. + interactsh *client.Client + // requests is a stored cache for interactsh-url->request-event data. + requests *ccache.Cache + + dotHostname string + eviction time.Duration + pollDuration time.Duration + cooldownDuration time.Duration +} + +// Options contains configuration options for interactsh nuclei integration. +type Options struct { + // ServerURL is the URL of the interactsh server. + ServerURL string + // CacheSize is the numbers of requests to keep track of at a time. + // Older items are discarded in LRU manner in favour of new requests. + CacheSize int64 + // Eviction is the period of time after which to automatically discard + // interaction requests. + Eviction time.Duration + // CooldownPeriod is additional time to wait for interactions after closing + // of the poller. + ColldownPeriod time.Duration + // PollDuration is the time to wait before each poll to the server for interactions. + PollDuration time.Duration + // Output is the output writer for nuclei + Output output.Writer + // Progress is the nuclei progress bar implementation. + Progress *progress.Progress +} + +// New returns a new interactsh server client +func New(options *Options) (*Client, error) { + parsed, err := url.Parse(options.ServerURL) + if err != nil { + return nil, errors.Wrap(err, "could not parse server url") + } + + interactsh, err := client.New(&client.Options{ + ServerURL: options.ServerURL, + PersistentSession: false, + }) + if err != nil { + return nil, errors.Wrap(err, "could not create client") + } + configure := ccache.Configure() + configure = configure.MaxSize(options.CacheSize) + cache := ccache.New(configure) + + client := &Client{ + interactsh: interactsh, + eviction: options.Eviction, + dotHostname: "." + parsed.Host, + requests: cache, + pollDuration: options.PollDuration, + cooldownDuration: options.ColldownPeriod, + } + client.interactsh.StartPolling(client.pollDuration, func(interaction *server.Interaction) { + item := client.requests.Get(interaction.UniqueID) + if item == nil { + return + } + data, ok := item.Value().(*internalRequestEvent) + if !ok { + return + } + client.requests.Delete(interaction.UniqueID) + + data.event.OperatorsResult = &operators.Result{ + Matches: map[string]struct{}{strings.ToLower(interaction.Protocol): {}}, + } + data.event.Results = data.makeResultFunc(data.event) + for _, result := range data.event.Results { + result.Interaction = interaction + options.Output.Write(result) + options.Progress.IncrementMatched() + } + }) + return client, nil +} + +// URL returns a new URL that can be interacted with +func (c *Client) URL() string { + return c.interactsh.URL() +} + +// Close closes the interactsh clients after waiting for cooldown period. +func (c *Client) Close() { + if c.cooldownDuration > 0 { + time.Sleep(c.cooldownDuration) + } + c.interactsh.StopPolling() + c.interactsh.Close() +} + +// MakeResultEventFunc is a result making function for nuclei +type MakeResultEventFunc func(wrapped *output.InternalWrappedEvent) []*output.ResultEvent + +type internalRequestEvent struct { + makeResultFunc MakeResultEventFunc + event *output.InternalWrappedEvent +} + +// RequestEvent is the event for a network request sent by nuclei. +func (c *Client) RequestEvent(interactshURL string, event *output.InternalWrappedEvent, makeResult MakeResultEventFunc) { + id := strings.TrimSuffix(interactshURL, c.dotHostname) + c.requests.Set(id, &internalRequestEvent{makeResultFunc: makeResult, event: event}, c.eviction) +} diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index a1d2c905..4ab93af7 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -67,7 +67,7 @@ type Request struct { options *protocols.ExecuterOptions attackType generators.Type totalRequests int - customHeaders []string + customHeaders map[string]string generator *generators.Generator // optional, only enabled when using payloads httpClient *retryablehttp.Client rawhttpClient *rawhttp.Client @@ -89,10 +89,15 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if err != nil { return errors.Wrap(err, "could not get dns client") } + r.customHeaders = make(map[string]string) r.httpClient = client r.options = options for _, option := range r.options.Options.CustomHeaders { - r.customHeaders = append(r.customHeaders, option) + parts := strings.SplitN(option, ":", 1) + if len(parts) != 2 { + continue + } + r.customHeaders[parts[0]] = strings.TrimSpace(parts[1]) } if len(r.Raw) > 0 { diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 27fb5768..b89660d2 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -59,7 +59,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { if unsafe && found { rawRequest.Headers[line] = "" } else { - rawRequest.Headers[key] = value + rawRequest.Headers[strings.TrimSpace(key)] = strings.TrimSpace(value) } } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 2755539d..1f154460 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -209,10 +209,7 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given if r.options.Options.RandomAgent { - builder := &strings.Builder{} - builder.WriteString("User-Agent: ") - builder.WriteString(uarand.GetRandom()) - r.customHeaders = append(r.customHeaders, builder.String()) + r.customHeaders["User-Agent"] = uarand.GetRandom() } r.setCustomHeaders(request) @@ -356,23 +353,11 @@ const two = 2 // setCustomHeaders sets the custom headers for generated request func (e *Request) setCustomHeaders(r *generatedRequest) { - for _, customHeader := range e.customHeaders { - if customHeader == "" { - continue - } - - // This should be pre-computed somewhere and done only once - tokens := strings.SplitN(customHeader, ":", two) - // if it's an invalid header skip it - if len(tokens) < 2 { - continue - } - - headerName, headerValue := tokens[0], strings.Join(tokens[1:], "") + for k, v := range e.customHeaders { if r.rawRequest != nil { - r.rawRequest.Headers[headerName] = headerValue + r.rawRequest.Headers[k] = v } else { - r.request.Header.Set(strings.TrimSpace(headerName), strings.TrimSpace(headerValue)) + r.request.Header.Set(strings.TrimSpace(k), strings.TrimSpace(v)) } } }