Merge pull request #479 from projectdiscovery/iceman-refactor

Multi-request condition support

v2/pkg/protocols/common/clusterer/executer.go

@@ -60,7 +60,7 @@ func (e *Executer) Execute(input string) (bool, error) {
 	var results bool
 
 	dynamicValues := make(map[string]interface{})
-	err := e.requests.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) {
+	err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) {
 		for _, operator := range e.operators {
 			result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract)
 			if matched && result != nil {
@@ -85,7 +85,7 @@ func (e *Executer) Execute(input string) (bool, error) {
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error {
 	dynamicValues := make(map[string]interface{})
-	_ = e.requests.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) {
+	_ = e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) {
 		for _, operator := range e.operators {
 			result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract)
 			if matched && result != nil {

v2/pkg/protocols/common/executer/executer.go

@@ -1,6 +1,8 @@
 package executer
 
 import (
+	"strings"
+
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 )
@@ -43,8 +45,20 @@ func (e *Executer) Execute(input string) (bool, error) {
 	var results bool
 
 	dynamicValues := make(map[string]interface{})
+	previous := make(map[string]interface{})
 	for _, req := range e.requests {
-		err := req.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) {
+		err := req.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
+			ID := req.GetID()
+			if ID != "" {
+				builder := &strings.Builder{}
+				for k, v := range event.InternalEvent {
+					builder.WriteString(ID)
+					builder.WriteString("_")
+					builder.WriteString(k)
+					previous[builder.String()] = v
+					builder.Reset()
+				}
+			}
 			if event.OperatorsResult == nil {
 				return
 			}
@@ -64,8 +78,21 @@ func (e *Executer) Execute(input string) (bool, error) {
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error {
 	dynamicValues := make(map[string]interface{})
+	previous := make(map[string]interface{})
+
 	for _, req := range e.requests {
-		_ = req.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) {
+		_ = req.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
+			ID := req.GetID()
+			if ID != "" {
+				builder := &strings.Builder{}
+				for k, v := range event.InternalEvent {
+					builder.WriteString(ID)
+					builder.WriteString("_")
+					builder.WriteString(k)
+					previous[builder.String()] = v
+					builder.Reset()
+				}
+			}
 			if event.OperatorsResult == nil {
 				return
 			}

v2/pkg/protocols/dns/dns.go

@@ -14,6 +14,8 @@ import (
 
 // Request contains a DNS protocol request to be made from a template
 type Request struct {
+	ID string `yaml:"id"`
+
 	// Recursion specifies whether to recurse all the answers.
 	Recursion bool `yaml:"recursion"`
 	// Path contains the path/s for the request
@@ -38,6 +40,11 @@ type Request struct {
 	options   *protocols.ExecuterOptions
 }
 
+// GetID returns the unique ID of the request if any.
+func (r *Request) GetID() string {
+	return r.ID
+}
+
 // Compile compiles the protocol request for further execution.
 func (r *Request) Compile(options *protocols.ExecuterOptions) error {
 	// Create a dns client for the class

v2/pkg/protocols/dns/request.go

@@ -14,7 +14,7 @@ import (
 var _ protocols.Request = &Request{}
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
-func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent, callback protocols.OutputEventCallback) error {
+func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	// Parse the URL and return domain if URL.
 	var domain string
 	if isURL(input) {
@@ -52,11 +52,14 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent
 		gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain)
 		fmt.Fprintf(os.Stderr, "%s\n", resp.String())
 	}
-	ouputEvent := r.responseToDSLMap(compiledRequest, resp, input, input)
+	outputEvent := r.responseToDSLMap(compiledRequest, resp, input, input)
+	for k, v := range previous {
+		outputEvent[k] = v
+	}
 
-	event := &output.InternalWrappedEvent{InternalEvent: ouputEvent}
+	event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
 	if r.CompiledOperators != nil {
-		result, ok := r.CompiledOperators.Execute(ouputEvent, r.Match, r.Extract)
+		result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract)
 		if ok && result != nil {
 			event.OperatorsResult = result
 			event.Results = r.MakeResultEvent(event)

v2/pkg/protocols/file/file.go

@@ -8,6 +8,8 @@ import (
 
 // Request contains a File matching mechanism for local disk operations.
 type Request struct {
+	ID string `yaml:"id"`
+
 	// MaxSize is the maximum size of the file to run request on.
 	// By default, nuclei will process 5MB files and not go more than that.
 	// It can be set to much lower or higher depending on use.
@@ -35,6 +37,11 @@ type Request struct {
 // defaultDenylist is the default list of extensions to be denied
 var defaultDenylist = []string{".3g2", ".3gp", ".7z", ".apk", ".arj", ".avi", ".axd", ".bmp", ".css", ".csv", ".deb", ".dll", ".doc", ".drv", ".eot", ".exe", ".flv", ".gif", ".gifv", ".gz", ".h264", ".ico", ".iso", ".jar", ".jpeg", ".jpg", ".lock", ".m4a", ".m4v", ".map", ".mkv", ".mov", ".mp3", ".mp4", ".mpeg", ".mpg", ".msi", ".ogg", ".ogm", ".ogv", ".otf", ".pdf", ".pkg", ".png", ".ppt", ".psd", ".rar", ".rm", ".rpm", ".svg", ".swf", ".sys", ".tar.gz", ".tar", ".tif", ".tiff", ".ttf", ".txt", ".vob", ".wav", ".webm", ".wmv", ".woff", ".woff2", ".xcf", ".xls", ".xlsx", ".zip"}
 
+// GetID returns the unique ID of the request if any.
+func (r *Request) GetID() string {
+	return r.ID
+}
+
 // Compile compiles the protocol request for further execution.
 func (r *Request) Compile(options *protocols.ExecuterOptions) error {
 	if len(r.Matchers) > 0 || len(r.Extractors) > 0 {

v2/pkg/protocols/file/request.go

@@ -16,7 +16,7 @@ import (
 var _ protocols.Request = &Request{}
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
-func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent, callback protocols.OutputEventCallback) error {
+func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	wg := sizedwaitgroup.New(r.options.Options.RateLimit)
 
 	err := r.getInputPaths(input, func(data string) {
@@ -54,11 +54,14 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent
 				fmt.Fprintf(os.Stderr, "%s\n", dataStr)
 			}
 			gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, data)
-			ouputEvent := r.responseToDSLMap(dataStr, input, data)
+			outputEvent := r.responseToDSLMap(dataStr, input, data)
+			for k, v := range previous {
+				outputEvent[k] = v
+			}
 
-			event := &output.InternalWrappedEvent{InternalEvent: ouputEvent}
+			event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
 			if r.CompiledOperators != nil {
-				result, ok := r.CompiledOperators.Execute(ouputEvent, r.Match, r.Extract)
+				result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract)
 				if ok && result != nil {
 					event.OperatorsResult = result
 					event.Results = r.MakeResultEvent(event)

v2/pkg/protocols/http/http.go

@@ -14,6 +14,8 @@ import (
 
 // Request contains a http request to be made from a template
 type Request struct {
+	ID string `yaml:"id"`
+
 	// Name is the name of the request
 	Name string `yaml:"Name"`
 	// AttackType is the attack type
@@ -71,6 +73,11 @@ type Request struct {
 	rawhttpClient *rawhttp.Client
 }
 
+// GetID returns the unique ID of the request if any.
+func (r *Request) GetID() string {
+	return r.ID
+}
+
 // Compile compiles the protocol request for further execution.
 func (r *Request) Compile(options *protocols.ExecuterOptions) error {
 	client, err := httpclientpool.Get(options.Options, &httpclientpool.Configuration{

v2/pkg/protocols/http/request.go

@@ -29,7 +29,7 @@ import (
 const defaultMaxWorkers = 150
 
 // executeRaceRequest executes race condition request for a URL
-func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]interface{}, callback protocols.OutputEventCallback) error {
+func (e *Request) executeRaceRequest(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	generator := e.newGenerator()
 
 	maxWorkers := e.RaceNumberRequests
@@ -45,7 +45,7 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]int
 	for i := 0; i < e.RaceNumberRequests; i++ {
 		swg.Add()
 		go func(httpRequest *generatedRequest) {
-			err := e.executeRequest(reqURL, httpRequest, dynamicValues, callback)
+			err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
 			mutex.Lock()
 			if err != nil {
 				requestErr = multierr.Append(requestErr, err)
@@ -59,7 +59,7 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]int
 }
 
 // executeRaceRequest executes race condition request for a URL
-func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]interface{}, callback protocols.OutputEventCallback) error {
+func (e *Request) executeParallelHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	generator := e.newGenerator()
 
 	// Workers that keeps enqueuing new requests
@@ -82,7 +82,7 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]in
 			defer swg.Done()
 
 			e.options.RateLimiter.Take()
-			err := e.executeRequest(reqURL, httpRequest, dynamicValues, callback)
+			err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
 			mutex.Lock()
 			if err != nil {
 				requestErr = multierr.Append(requestErr, err)
@@ -96,7 +96,7 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]in
 }
 
 // executeRaceRequest executes race condition request for a URL
-func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]interface{}, callback protocols.OutputEventCallback) error {
+func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	generator := e.newGenerator()
 
 	// need to extract the target from the url
@@ -141,7 +141,7 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter
 		go func(httpRequest *generatedRequest) {
 			defer swg.Done()
 
-			err := e.executeRequest(reqURL, httpRequest, dynamicValues, callback)
+			err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
 			mutex.Lock()
 			if err != nil {
 				requestErr = multierr.Append(requestErr, err)
@@ -155,20 +155,20 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter
 }
 
 // ExecuteWithResults executes the final request on a URL
-func (r *Request) ExecuteWithResults(reqURL string, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error {
+func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	// verify if pipeline was requested
 	if r.Pipeline {
-		return r.executeTurboHTTP(reqURL, dynamicValues, callback)
+		return r.executeTurboHTTP(reqURL, dynamicValues, previous, callback)
 	}
 
 	// verify if a basic race condition was requested
 	if r.Race && r.RaceNumberRequests > 0 {
-		return r.executeRaceRequest(reqURL, dynamicValues, callback)
+		return r.executeRaceRequest(reqURL, dynamicValues, previous, callback)
 	}
 
 	// verify if parallel elaboration was requested
 	if r.Threads > 0 {
-		return r.executeParallelHTTP(reqURL, dynamicValues, callback)
+		return r.executeParallelHTTP(reqURL, dynamicValues, previous, callback)
 	}
 
 	generator := r.newGenerator()
@@ -186,7 +186,7 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues output.Interna
 
 		var gotOutput bool
 		r.options.RateLimiter.Take()
-		err = r.executeRequest(reqURL, request, dynamicValues, func(event *output.InternalWrappedEvent) {
+		err = r.executeRequest(reqURL, request, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
 			// Add the extracts to the dynamic values if any.
 			if event.OperatorsResult != nil {
 				gotOutput = true
@@ -208,7 +208,7 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues output.Interna
 }
 
 // executeRequest executes the actual generated request and returns error if occured
-func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues map[string]interface{}, callback protocols.OutputEventCallback) error {
+func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	// Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given
 	if r.options.Options.RandomAgent {
 		builder := &strings.Builder{}
@@ -337,6 +337,9 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam
 	}
 	outputEvent := r.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, request.meta)
 	outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
+	for k, v := range previous {
+		outputEvent[k] = v
+	}
 
 	event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
 	if r.CompiledOperators != nil {

v2/pkg/protocols/network/network.go

@@ -13,6 +13,8 @@ import (
 
 // Request contains a Network protocol request to be made from a template
 type Request struct {
+	ID string `yaml:"id"`
+
 	// Address is the address to send requests to (host:port combos generally)
 	Address   []string `yaml:"host"`
 	addresses []keyValue
@@ -45,6 +47,11 @@ type Input struct {
 	Type string `yaml:"type"`
 }
 
+// GetID returns the unique ID of the request if any.
+func (r *Request) GetID() string {
+	return r.ID
+}
+
 // Compile compiles the protocol request for further execution.
 func (r *Request) Compile(options *protocols.ExecuterOptions) error {
 	var err error

v2/pkg/protocols/network/request.go

@@ -20,7 +20,7 @@ import (
 var _ protocols.Request = &Request{}
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
-func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent, callback protocols.OutputEventCallback) error {
+func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	address, err := getAddress(input)
 	if err != nil {
 		r.options.Output.Request(r.options.TemplateID, input, "network", err)
@@ -38,7 +38,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent
 			actualAddress = net.JoinHostPort(actualAddress, kv.value)
 		}
 
-		err = r.executeAddress(actualAddress, address, input, callback)
+		err = r.executeAddress(actualAddress, address, input, previous, callback)
 		if err != nil {
 			gologger.Verbose().Lable("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err)
 			continue
@@ -48,7 +48,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent
 }
 
 // executeAddress executes the request for an address
-func (r *Request) executeAddress(actualAddress, address, input string, callback protocols.OutputEventCallback) error {
+func (r *Request) executeAddress(actualAddress, address, input string, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	if !strings.Contains(actualAddress, ":") {
 		err := errors.New("no port provided in network protocol request")
 		r.options.Output.Request(r.options.TemplateID, address, "network", err)
@@ -125,6 +125,9 @@ func (r *Request) executeAddress(actualAddress, address, input string, callback
 	}
 	outputEvent := r.responseToDSLMap(reqBuilder.String(), resp, input, actualAddress)
 	outputEvent["ip"] = r.dialer.GetDialedIP(hostname)
+	for k, v := range previous {
+		outputEvent[k] = v
+	}
 
 	event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
 	if r.CompiledOperators != nil {

v2/pkg/protocols/protocols.go

@@ -51,12 +51,16 @@ type Request interface {
 	Compile(options *ExecuterOptions) error
 	// Requests returns the total number of requests the rule will perform
 	Requests() int
+	// GetID returns the ID for the request if any. IDs are used for multi-request
+	// condition matching. So, two requests can be sent and their match can
+	// be evaluated from the third request by using the IDs for both requests.
+	GetID() string
 	// Match performs matching operation for a matcher on model and returns true or false.
 	Match(data map[string]interface{}, matcher *matchers.Matcher) bool
 	// Extract performs extracting operation for a extractor on model and returns true or false.
 	Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
 	// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
-	ExecuteWithResults(input string, metadata output.InternalEvent, callback OutputEventCallback) error
+	ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error
 }
 
 // OutputEventCallback is a callback event for any results found during scanning.