From 827fd99f018764c838f3fb0632b98355bf0fdcef Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 16 Jan 2021 14:10:24 +0530 Subject: [PATCH] Added multi-request condition support --- v2/pkg/protocols/common/clusterer/executer.go | 4 +-- v2/pkg/protocols/common/executer/executer.go | 31 +++++++++++++++++-- v2/pkg/protocols/dns/dns.go | 7 +++++ v2/pkg/protocols/dns/request.go | 11 ++++--- v2/pkg/protocols/file/file.go | 7 +++++ v2/pkg/protocols/file/request.go | 11 ++++--- v2/pkg/protocols/http/http.go | 7 +++++ v2/pkg/protocols/http/request.go | 27 +++++++++------- v2/pkg/protocols/network/network.go | 7 +++++ v2/pkg/protocols/network/request.go | 9 ++++-- v2/pkg/protocols/protocols.go | 6 +++- 11 files changed, 99 insertions(+), 28 deletions(-) diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/protocols/common/clusterer/executer.go index 1a2a0508..0d78b00e 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/protocols/common/clusterer/executer.go @@ -60,7 +60,7 @@ func (e *Executer) Execute(input string) (bool, error) { var results bool dynamicValues := make(map[string]interface{}) - err := e.requests.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) { + err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract) if matched && result != nil { @@ -85,7 +85,7 @@ func (e *Executer) Execute(input string) (bool, error) { // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { dynamicValues := make(map[string]interface{}) - _ = e.requests.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) { + _ = e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract) if matched && result != nil { diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index a06bd0d7..ff2ee0f1 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -1,6 +1,8 @@ package executer import ( + "strings" + "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" ) @@ -43,8 +45,20 @@ func (e *Executer) Execute(input string) (bool, error) { var results bool dynamicValues := make(map[string]interface{}) + previous := make(map[string]interface{}) for _, req := range e.requests { - err := req.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) { + err := req.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { + ID := req.GetID() + if ID != "" { + builder := &strings.Builder{} + for k, v := range event.InternalEvent { + builder.WriteString(ID) + builder.WriteString("_") + builder.WriteString(k) + previous[builder.String()] = v + builder.Reset() + } + } if event.OperatorsResult == nil { return } @@ -64,8 +78,21 @@ func (e *Executer) Execute(input string) (bool, error) { // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { dynamicValues := make(map[string]interface{}) + previous := make(map[string]interface{}) + for _, req := range e.requests { - _ = req.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) { + _ = req.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { + ID := req.GetID() + if ID != "" { + builder := &strings.Builder{} + for k, v := range event.InternalEvent { + builder.WriteString(ID) + builder.WriteString("_") + builder.WriteString(k) + previous[builder.String()] = v + builder.Reset() + } + } if event.OperatorsResult == nil { return } diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 275d8126..464b7754 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -14,6 +14,8 @@ import ( // Request contains a DNS protocol request to be made from a template type Request struct { + ID string `yaml:"id"` + // Recursion specifies whether to recurse all the answers. Recursion bool `yaml:"recursion"` // Path contains the path/s for the request @@ -38,6 +40,11 @@ type Request struct { options *protocols.ExecuterOptions } +// GetID returns the unique ID of the request if any. +func (r *Request) GetID() string { + return r.ID +} + // Compile compiles the protocol request for further execution. func (r *Request) Compile(options *protocols.ExecuterOptions) error { // Create a dns client for the class diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 7b95cad4..00b97bd9 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -14,7 +14,7 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent, callback protocols.OutputEventCallback) error { +func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // Parse the URL and return domain if URL. var domain string if isURL(input) { @@ -52,11 +52,14 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain) fmt.Fprintf(os.Stderr, "%s\n", resp.String()) } - ouputEvent := r.responseToDSLMap(compiledRequest, resp, input, input) + outputEvent := r.responseToDSLMap(compiledRequest, resp, input, input) + for k, v := range previous { + outputEvent[k] = v + } - event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(ouputEvent, r.Match, r.Extract) + result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) if ok && result != nil { event.OperatorsResult = result event.Results = r.MakeResultEvent(event) diff --git a/v2/pkg/protocols/file/file.go b/v2/pkg/protocols/file/file.go index 0056a271..6708b39b 100644 --- a/v2/pkg/protocols/file/file.go +++ b/v2/pkg/protocols/file/file.go @@ -8,6 +8,8 @@ import ( // Request contains a File matching mechanism for local disk operations. type Request struct { + ID string `yaml:"id"` + // MaxSize is the maximum size of the file to run request on. // By default, nuclei will process 5MB files and not go more than that. // It can be set to much lower or higher depending on use. @@ -35,6 +37,11 @@ type Request struct { // defaultDenylist is the default list of extensions to be denied var defaultDenylist = []string{".3g2", ".3gp", ".7z", ".apk", ".arj", ".avi", ".axd", ".bmp", ".css", ".csv", ".deb", ".dll", ".doc", ".drv", ".eot", ".exe", ".flv", ".gif", ".gifv", ".gz", ".h264", ".ico", ".iso", ".jar", ".jpeg", ".jpg", ".lock", ".m4a", ".m4v", ".map", ".mkv", ".mov", ".mp3", ".mp4", ".mpeg", ".mpg", ".msi", ".ogg", ".ogm", ".ogv", ".otf", ".pdf", ".pkg", ".png", ".ppt", ".psd", ".rar", ".rm", ".rpm", ".svg", ".swf", ".sys", ".tar.gz", ".tar", ".tif", ".tiff", ".ttf", ".txt", ".vob", ".wav", ".webm", ".wmv", ".woff", ".woff2", ".xcf", ".xls", ".xlsx", ".zip"} +// GetID returns the unique ID of the request if any. +func (r *Request) GetID() string { + return r.ID +} + // Compile compiles the protocol request for further execution. func (r *Request) Compile(options *protocols.ExecuterOptions) error { if len(r.Matchers) > 0 || len(r.Extractors) > 0 { diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index f9f3e537..13d4df9c 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -16,7 +16,7 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent, callback protocols.OutputEventCallback) error { +func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { wg := sizedwaitgroup.New(r.options.Options.RateLimit) err := r.getInputPaths(input, func(data string) { @@ -54,11 +54,14 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent fmt.Fprintf(os.Stderr, "%s\n", dataStr) } gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, data) - ouputEvent := r.responseToDSLMap(dataStr, input, data) + outputEvent := r.responseToDSLMap(dataStr, input, data) + for k, v := range previous { + outputEvent[k] = v + } - event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(ouputEvent, r.Match, r.Extract) + result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) if ok && result != nil { event.OperatorsResult = result event.Results = r.MakeResultEvent(event) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index bebd26b3..a1d2c905 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -14,6 +14,8 @@ import ( // Request contains a http request to be made from a template type Request struct { + ID string `yaml:"id"` + // Name is the name of the request Name string `yaml:"Name"` // AttackType is the attack type @@ -71,6 +73,11 @@ type Request struct { rawhttpClient *rawhttp.Client } +// GetID returns the unique ID of the request if any. +func (r *Request) GetID() string { + return r.ID +} + // Compile compiles the protocol request for further execution. func (r *Request) Compile(options *protocols.ExecuterOptions) error { client, err := httpclientpool.Get(options.Options, &httpclientpool.Configuration{ diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 7ec702be..23773f66 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -29,7 +29,7 @@ import ( const defaultMaxWorkers = 150 // executeRaceRequest executes race condition request for a URL -func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]interface{}, callback protocols.OutputEventCallback) error { +func (e *Request) executeRaceRequest(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { generator := e.newGenerator() maxWorkers := e.RaceNumberRequests @@ -45,7 +45,7 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]int for i := 0; i < e.RaceNumberRequests; i++ { swg.Add() go func(httpRequest *generatedRequest) { - err := e.executeRequest(reqURL, httpRequest, dynamicValues, callback) + err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) @@ -59,7 +59,7 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]int } // executeRaceRequest executes race condition request for a URL -func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]interface{}, callback protocols.OutputEventCallback) error { +func (e *Request) executeParallelHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { generator := e.newGenerator() // Workers that keeps enqueuing new requests @@ -82,7 +82,7 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]in defer swg.Done() e.options.RateLimiter.Take() - err := e.executeRequest(reqURL, httpRequest, dynamicValues, callback) + err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) @@ -96,7 +96,7 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]in } // executeRaceRequest executes race condition request for a URL -func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]interface{}, callback protocols.OutputEventCallback) error { +func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { generator := e.newGenerator() // need to extract the target from the url @@ -141,7 +141,7 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter go func(httpRequest *generatedRequest) { defer swg.Done() - err := e.executeRequest(reqURL, httpRequest, dynamicValues, callback) + err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) @@ -155,20 +155,20 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter } // ExecuteWithResults executes the final request on a URL -func (r *Request) ExecuteWithResults(reqURL string, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error { +func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // verify if pipeline was requested if r.Pipeline { - return r.executeTurboHTTP(reqURL, dynamicValues, callback) + return r.executeTurboHTTP(reqURL, dynamicValues, previous, callback) } // verify if a basic race condition was requested if r.Race && r.RaceNumberRequests > 0 { - return r.executeRaceRequest(reqURL, dynamicValues, callback) + return r.executeRaceRequest(reqURL, dynamicValues, previous, callback) } // verify if parallel elaboration was requested if r.Threads > 0 { - return r.executeParallelHTTP(reqURL, dynamicValues, callback) + return r.executeParallelHTTP(reqURL, dynamicValues, previous, callback) } generator := r.newGenerator() @@ -186,7 +186,7 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues output.Interna var gotOutput bool r.options.RateLimiter.Take() - err = r.executeRequest(reqURL, request, dynamicValues, func(event *output.InternalWrappedEvent) { + err = r.executeRequest(reqURL, request, dynamicValues, previous, func(event *output.InternalWrappedEvent) { // Add the extracts to the dynamic values if any. if event.OperatorsResult != nil { gotOutput = true @@ -208,7 +208,7 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues output.Interna } // executeRequest executes the actual generated request and returns error if occured -func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues map[string]interface{}, callback protocols.OutputEventCallback) error { +func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given if r.options.Options.RandomAgent { builder := &strings.Builder{} @@ -337,6 +337,9 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam } outputEvent := r.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, request.meta) outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) + for k, v := range previous { + outputEvent[k] = v + } event := &output.InternalWrappedEvent{InternalEvent: outputEvent} if r.CompiledOperators != nil { diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index a99f3e07..06d0447d 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -13,6 +13,8 @@ import ( // Request contains a Network protocol request to be made from a template type Request struct { + ID string `yaml:"id"` + // Address is the address to send requests to (host:port combos generally) Address []string `yaml:"host"` addresses []keyValue @@ -45,6 +47,11 @@ type Input struct { Type string `yaml:"type"` } +// GetID returns the unique ID of the request if any. +func (r *Request) GetID() string { + return r.ID +} + // Compile compiles the protocol request for further execution. func (r *Request) Compile(options *protocols.ExecuterOptions) error { var err error diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 968770c1..c71eb83b 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -20,7 +20,7 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent, callback protocols.OutputEventCallback) error { +func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { address, err := getAddress(input) if err != nil { r.options.Output.Request(r.options.TemplateID, input, "network", err) @@ -38,7 +38,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent actualAddress = net.JoinHostPort(actualAddress, kv.value) } - err = r.executeAddress(actualAddress, address, input, callback) + err = r.executeAddress(actualAddress, address, input, previous, callback) if err != nil { gologger.Verbose().Lable("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err) continue @@ -48,7 +48,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent } // executeAddress executes the request for an address -func (r *Request) executeAddress(actualAddress, address, input string, callback protocols.OutputEventCallback) error { +func (r *Request) executeAddress(actualAddress, address, input string, previous output.InternalEvent, callback protocols.OutputEventCallback) error { if !strings.Contains(actualAddress, ":") { err := errors.New("no port provided in network protocol request") r.options.Output.Request(r.options.TemplateID, address, "network", err) @@ -125,6 +125,9 @@ func (r *Request) executeAddress(actualAddress, address, input string, callback } outputEvent := r.responseToDSLMap(reqBuilder.String(), resp, input, actualAddress) outputEvent["ip"] = r.dialer.GetDialedIP(hostname) + for k, v := range previous { + outputEvent[k] = v + } event := &output.InternalWrappedEvent{InternalEvent: outputEvent} if r.CompiledOperators != nil { diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index c808d658..5a40a54d 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -51,12 +51,16 @@ type Request interface { Compile(options *ExecuterOptions) error // Requests returns the total number of requests the rule will perform Requests() int + // GetID returns the ID for the request if any. IDs are used for multi-request + // condition matching. So, two requests can be sent and their match can + // be evaluated from the third request by using the IDs for both requests. + GetID() string // Match performs matching operation for a matcher on model and returns true or false. Match(data map[string]interface{}, matcher *matchers.Matcher) bool // Extract performs extracting operation for a extractor on model and returns true or false. Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. - ExecuteWithResults(input string, metadata output.InternalEvent, callback OutputEventCallback) error + ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error } // OutputEventCallback is a callback event for any results found during scanning.