Adding support for constants (#3692)

* adding support for constants

* fixing typo

* adding integration test

* fixing lint issues

* fixing template syntax

integration_tests/http/cli-with-constants.yaml

@@ -0,0 +1,18 @@
+id: cli-with-constants
+
+info:
+  name: Cli Var with Constants
+  author: pdteam
+  severity: info
+
+constants:
+  test: test-in-template
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}?p={{test}}"
+    matchers:
+      - type: word
+        words:
+          - "test-in-template"

v2/cmd/integration-test/http.go

@@ -77,6 +77,7 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/cl-body-without-header.yaml":              &httpCLBodyWithoutHeader{},
 	"http/cl-body-with-header.yaml":                 &httpCLBodyWithHeader{},
 	"http/save-extractor-values-to-file.yaml":       &httpSaveExtractorValuesToFile{},
+	"http/cli-with-constants.yaml":                  &ConstantWithCliVar{},
 }
 
 type httpInteractshRequest struct{}
@@ -1403,3 +1404,22 @@ func (h *httpSaveExtractorValuesToFile) Execute(filePath string) error {
 	}
 	return expectResultsCount(results, 1)
 }
+
+// constant shouldn't be overwritten by cli var with same name
+type ConstantWithCliVar struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *ConstantWithCliVar) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprint(w, r.URL.Query().Get("p"))
+	})
+	ts := httptest.NewTLSServer(router)
+	defer ts.Close()
+
+	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-V", "test=fromcli")
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(got, 1)
+}

v2/pkg/protocols/common/variables/doc.go

@@ -6,6 +6,7 @@ package variables
 // 3. OptionsMap   - Variables passed using CLI Options (+ Env) (available at generators.BuildPayloadFromOptions)
 // 4. DynamicMap   - Variables Obtained by extracting data from templates  (available at Request.ExecuteWithResults + merged with previous internalEvent)
 // 5. ProtocolMap  - Variables generated by Evaluation Request / Responses of xyz protocol (available in Request.Make)
+// 6. ConstantsMap  - Constants defined in the template (available at Request.options.Constants in protocols)
 
 // As we can tell , all variables sources are not linear i.e why they need to re-evaluated
 // consider example
@@ -22,3 +23,5 @@ package variables
 // 1. VariablesMap
 // 2. PayloadsMap
 // Everytime Linear Sources are updated , Non-Linear Sources need to be re-evaluated
+
+// Constants (no need to re-evaluate, should contain only scalars)

v2/pkg/protocols/dns/request.go

@@ -55,7 +55,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 	// merge with metadata (eg. from workflow context)
 	vars = generators.MergeMaps(vars, metadata, optionVars)
 	variablesMap := request.options.Variables.Evaluate(vars)
-	vars = generators.MergeMaps(vars, variablesMap)
+	vars = generators.MergeMaps(vars, variablesMap, request.options.Constants)
 
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()

v2/pkg/protocols/headless/engine/http_client.go

@@ -64,10 +64,8 @@ func newHttpClient(options *types.Options) (*http.Client, error) {
 		dc := dialer.(interface {
 			DialContext(ctx context.Context, network, addr string) (net.Conn, error)
 		})
-		if proxyErr == nil {
 		transport.DialContext = dc.DialContext
 	}
-	}
 
 	jar, _ := cookiejar.New(nil)
 

v2/pkg/protocols/headless/request.go

@@ -41,7 +41,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 	payloads := generators.BuildPayloadFromOptions(request.options.Options)
 	values := generators.MergeMaps(vars, metadata, payloads)
 	variablesMap := request.options.Variables.Evaluate(values)
-	payloads = generators.MergeMaps(variablesMap, payloads)
+	payloads = generators.MergeMaps(variablesMap, payloads, request.options.Constants)
 
 	// check for operator matches by wrapping callback
 	gotmatches := false

v2/pkg/protocols/http/build_request.go

@@ -112,7 +112,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
 		r.interactshURLs = append(r.interactshURLs, interactURLs...)
 	}
 	// allVars contains all variables from all sources
-	allVars := generators.MergeMaps(dynamicValues, defaultReqVars, optionVars, variablesMap)
+	allVars := generators.MergeMaps(dynamicValues, defaultReqVars, optionVars, variablesMap, r.options.Constants)
 
 	// Evaluate payload variables
 	// eg: payload variables can be username: jon.doe@{{Hostname}}
@@ -170,10 +170,10 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
 
 	signerVars := GetDefaultSignerVars(r.request.Signature.Value)
 	// this will ensure that default signer variables are overwritten by other variables
-	values = generators.MergeMaps(signerVars, values)
+	values = generators.MergeMaps(signerVars, values, r.options.Constants)
 
 	// priority of variables is as follows (from low to high) for self contained templates
-	// default signer vars < variables <  cli vars  < payload < dynamic values
+	// default signer vars < variables <  cli vars  < payload < dynamic values < constants
 
 	// evaluate request
 	data, err := expressions.Evaluate(data, values)

v2/pkg/protocols/http/fuzz/execute.go

@@ -47,7 +47,7 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) error {
 	baseValues := input.Values
 	if rule.generator == nil {
 		evaluatedValues, interactURLs := rule.options.Variables.EvaluateWithInteractsh(baseValues, rule.options.Interactsh)
-		input.Values = generators.MergeMaps(evaluatedValues, baseValues)
+		input.Values = generators.MergeMaps(evaluatedValues, baseValues, rule.options.Constants)
 		input.InteractURLs = interactURLs
 		err := rule.executeRuleValues(input)
 		return err
@@ -60,7 +60,7 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) error {
 		}
 		evaluatedValues, interactURLs := rule.options.Variables.EvaluateWithInteractsh(generators.MergeMaps(values, baseValues), rule.options.Interactsh)
 		input.InteractURLs = interactURLs
-		input.Values = generators.MergeMaps(values, evaluatedValues, baseValues)
+		input.Values = generators.MergeMaps(values, evaluatedValues, baseValues, rule.options.Constants)
 
 		if err := rule.executeRuleValues(input); err != nil {
 			return err

v2/pkg/protocols/http/request.go

@@ -318,7 +318,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
 func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	if request.Pipeline || request.Race && request.RaceNumberRequests > 0 || request.Threads > 0 {
 		variablesMap := request.options.Variables.Evaluate(generators.MergeMaps(dynamicValues, previous))
-		dynamicValues = generators.MergeMaps(variablesMap, dynamicValues)
+		dynamicValues = generators.MergeMaps(variablesMap, dynamicValues, request.options.Constants)
 	}
 	// verify if pipeline was requested
 	if request.Pipeline {
@@ -638,7 +638,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 	if !request.Unsafe && resp != nil && generatedRequest.request != nil && resp.Request != nil && !request.Race {
 		bodyBytes, _ := generatedRequest.request.BodyBytes()
 		resp.Request.Body = io.NopCloser(bytes.NewReader(bodyBytes))
-		command, _ := http2curl.GetCurlCommand(resp.Request)
+		command, err := http2curl.GetCurlCommand(resp.Request)
 		if err == nil && command != nil {
 			curlCommand = command.String()
 		}

v2/pkg/protocols/network/request.go

@@ -38,7 +38,7 @@ func (request *Request) Type() templateTypes.ProtocolType {
 }
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
-func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
+func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	var address string
 	var err error
 
@@ -54,7 +54,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata
 	}
 	variables := protocolutils.GenerateVariables(address, false, nil)
 	variablesMap := request.options.Variables.Evaluate(variables)
-	variables = generators.MergeMaps(variablesMap, variables)
+	variables = generators.MergeMaps(variablesMap, variables, request.options.Constants)
 
 	for _, kv := range request.addresses {
 		actualAddress := replacer.Replace(kv.address, variables)

v2/pkg/protocols/protocols.go

@@ -70,6 +70,8 @@ type ExecuterOptions struct {
 	StopAtFirstMatch bool
 	// Variables is a list of variables from template
 	Variables variables.Variable
+	// Constants is a list of constants from template
+	Constants map[string]interface{}
 	// ExcludeMatchers is the list of matchers to exclude
 	ExcludeMatchers *excludematchers.ExcludeMatchers
 	// InputHelper is a helper for input normalization

v2/pkg/protocols/ssl/ssl.go

@@ -190,7 +190,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 	hostnameVariables := protocolutils.GenerateDNSVariables(hostname)
 	values := generators.MergeMaps(payloadValues, hostnameVariables)
 	variablesMap := request.options.Variables.Evaluate(values)
-	payloadValues = generators.MergeMaps(variablesMap, payloadValues)
+	payloadValues = generators.MergeMaps(variablesMap, payloadValues, request.options.Constants)
 
 	if vardump.EnableVarDump {
 		gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues))

v2/pkg/protocols/websocket/websocket.go

@@ -176,7 +176,7 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam
 	defaultVars := protocolutils.GenerateVariables(parsed, false, nil)
 	optionVars := generators.BuildPayloadFromOptions(request.options.Options)
 	variables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues))
-	payloadValues := generators.MergeMaps(variables, defaultVars, optionVars, dynamicValues)
+	payloadValues := generators.MergeMaps(variables, defaultVars, optionVars, dynamicValues, request.options.Constants)
 
 	requestOptions := request.options
 	for key, value := range request.Headers {

v2/pkg/protocols/whois/whois.go

@@ -92,7 +92,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 	optionVars := generators.BuildPayloadFromOptions(request.options.Options)
 	vars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues))
 
-	variables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues)
+	variables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues, request.options.Constants)
 
 	if vardump.EnableVarDump {
 		gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(variables))

v2/pkg/templates/compile.go

@@ -232,6 +232,8 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option
 		options.Variables = template.Variables
 	}
 
+	options.Constants = template.Constants
+
 	// If no requests, and it is also not a workflow, return error.
 	if template.Requests() == 0 {
 		return nil, fmt.Errorf("no requests defined for %s", template.ID)

v2/pkg/templates/templates.go

@@ -113,6 +113,10 @@ type Template struct {
 	//   Variables contains any variables for the current request.
 	Variables variables.Variable `yaml:"variables,omitempty" json:"variables,omitempty" jsonschema:"title=variables for the http request,description=Variables contains any variables for the current request"`
 
+	// description: |
+	//   Constants contains any scalar costant for the current template
+	Constants map[string]interface{} `yaml:"constants,omitempty" json:"constants,omitempty" jsonschema:"title=constant for the template,description=constants contains any constant for the template"`
+
 	// TotalRequests is the total number of requests for the template.
 	TotalRequests int `yaml:"-" json:"-"`
 	// Executer is the actual template executor for running template requests