Merge pull request #938 from projectdiscovery/feature/global-payload#554

Add payload, helper function support to all http requests

v2/pkg/protocols/http/build_request.go

@@ -16,7 +16,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw"
 	"github.com/projectdiscovery/rawhttp"
@@ -75,7 +74,7 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
 	if isRawRequest {
 		return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL)
 	}
-	return r.makeHTTPRequestFromModel(ctx, data, values, interactURL)
+	return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL)
 }
 
 // Total returns the total number of requests for the generator
@@ -104,23 +103,38 @@ func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) {
 }
 
 // MakeHTTPRequestFromModel creates a *http.Request from a request template
-func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}, interactURL string) (*generatedRequest, error) {
-	final := replacer.Replace(data, values)
+func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values, generatorValues map[string]interface{}, interactURL string) (*generatedRequest, error) {
 	if interactURL != "" {
-		final = r.options.Interactsh.ReplaceMarkers(final, interactURL)
+		data = r.options.Interactsh.ReplaceMarkers(data, interactURL)
+	}
+
+	// Combine the template payloads along with base
+	// request values.
+	finalValues := generators.MergeMaps(generatorValues, values)
+
+	// Evaulate the expressions for the request if any.
+	var err error
+	data, err = expressions.Evaluate(data, finalValues)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not evaluate helper expressions")
+	}
+
+	method, err := expressions.Evaluate(r.request.Method, finalValues)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not evaluate helper expressions")
 	}
 
 	// Build a request on the specified URL
-	req, err := http.NewRequestWithContext(ctx, r.request.Method, final, nil)
+	req, err := http.NewRequestWithContext(ctx, method, data, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	request, err := r.fillRequest(req, values, interactURL)
+	request, err := r.fillRequest(req, finalValues, interactURL)
 	if err != nil {
 		return nil, err
 	}
-	return &generatedRequest{request: request, original: r.request}, nil
+	return &generatedRequest{request: request, meta: generatorValues, original: r.request}, nil
 }
 
 // makeHTTPRequestFromRaw creates a *http.Request from a raw request
@@ -176,7 +190,7 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest
 			req.Host = value
 		}
 	}
-	request, err := r.fillRequest(req, values, "")
+	request, err := r.fillRequest(req, finalValues, "")
 	if err != nil {
 		return nil, err
 	}
@@ -191,9 +205,13 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte
 		if interactURL != "" {
 			value = r.options.Interactsh.ReplaceMarkers(value, interactURL)
 		}
-		req.Header[header] = []string{replacer.Replace(value, values)}
+		value, err := expressions.Evaluate(value, values)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not evaluate helper expressions")
+		}
+		req.Header[header] = []string{value}
 		if header == "Host" {
-			req.Host = replacer.Replace(value, values)
+			req.Host = value
 		}
 	}
 
@@ -208,6 +226,10 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte
 		if interactURL != "" {
 			body = r.options.Interactsh.ReplaceMarkers(body, interactURL)
 		}
+		body, err := expressions.Evaluate(body, values)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not evaluate helper expressions")
+		}
 		req.Body = ioutil.NopCloser(strings.NewReader(body))
 	}
 	setHeader(req, "User-Agent", uarand.GetRandom())

v2/pkg/protocols/http/request_generator.go

@@ -31,20 +31,39 @@ func (r *Request) newGenerator() *requestGenerator {
 // nextValue returns the next path or the next raw request depending on user input
 // It returns false if all the inputs have been exhausted by the generator instance.
 func (r *requestGenerator) nextValue() (value string, payloads map[string]interface{}, result bool) {
-	// If we have paths, return the next path.
+	// For both raw/path requests, start with the request at current index.
+	// If we are not at the start, then check if the iterator for payloads
+	// has finished if there are any.
+	//
+	// If the iterator has finished for the current request
+	// then reset it and move on to the next value, otherwise use the last request.
+
 	if len(r.request.Path) > 0 && r.currentIndex < len(r.request.Path) {
+		if r.payloadIterator != nil {
+			payload, ok := r.payloadIterator.Value()
+			if !ok {
+				r.currentIndex++
+				r.payloadIterator.Reset()
+
+				// No more payloads request for us now.
+				if len(r.request.Path) == r.currentIndex {
+					return "", nil, false
+				}
+				if item := r.request.Path[r.currentIndex]; item != "" {
+					newPayload, ok := r.payloadIterator.Value()
+					return item, newPayload, ok
+				}
+				return "", nil, false
+			}
+			return r.request.Path[r.currentIndex], payload, true
+		}
 		if value := r.request.Path[r.currentIndex]; value != "" {
 			r.currentIndex++
 			return value, nil, true
 		}
 	}
 
-	// If we have raw requests, start with the request at current index.
-	// If we are not at the start, then check if the iterator for payloads
-	// has finished if there are any.
-	//
-	// If the iterator has finished for the current raw request
-	// then reset it and move on to the next value, otherwise use the last request.
+	
 	if len(r.request.Raw) > 0 && r.currentIndex < len(r.request.Raw) {
 		if r.payloadIterator != nil {
 			payload, ok := r.payloadIterator.Value()