Merge branch 'dev' into stop-at-first-match

integration_tests/http/dsl-matcher-variable.yaml

@@ -0,0 +1,23 @@
+id: dsl-matcher-variable
+
+info:
+  name: dsl-matcher-variable
+  author: pd-team
+  severity: info
+
+requests: 
+  - 
+    path: 
+      - "{{BaseURL}}"
+    payloads: 
+      VALUES: 
+        - This
+        - is
+        - test
+        - matcher
+        - text
+    matchers: 
+      - 
+        dsl: 
+          - 'contains(body,"{{VALUES}}")'
+        type: dsl

v2/cmd/integration-test/http.go

@@ -36,6 +36,7 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/get-case-insensitive.yaml":               &httpGetCaseInsensitive{},
 	"http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{},
 	"http/get-redirects-chain-headers.yaml":        &httpGetRedirectsChainHeaders{},
+	"http/dsl-matcher-variable.yaml":               &httpDSLVariable{},
 }
 
 type httpInteractshRequest struct{}
@@ -155,6 +156,27 @@ func (h *httpGet) Execute(filePath string) error {
 	return nil
 }
 
+type httpDSLVariable struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpDSLVariable) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "This is test matcher text")
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 5 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
 type httpPostBody struct{}
 
 // Execute executes a test case and returns an error if occurred

v2/internal/runner/options.go

@@ -116,7 +116,7 @@ func validateOptions(options *types.Options) error {
 // configureOutput configures the output logging levels to be displayed on the screen
 func configureOutput(options *types.Options) {
 	// If the user desires verbose output, show verbose output
-	if options.Verbose {
+	if options.Verbose || options.Validate {
 		gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
 	}
 	if options.Debug {

v2/internal/runner/runner.go

@@ -69,6 +69,8 @@ func New(options *types.Options) (*Runner, error) {
 	}
 	if options.Validate {
 		parsers.ShouldValidate = true
+		// Does not update the templates when validate flag is used
+		options.NoUpdateTemplates = true
 	}
 	if err := runner.updateTemplates(); err != nil {
 		gologger.Warning().Msgf("Could not update templates: %s\n", err)
@@ -231,10 +233,12 @@ func (r *Runner) RunEnumeration() error {
 		}
 		r.options.Templates = append(r.options.Templates, templatesLoaded...)
 	}
-	ignoreFile := config.ReadIgnoreFile()
-	r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
-	r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
-
+	// Exclude ignored file for validation
+	if !r.options.Validate {
+		ignoreFile := config.ReadIgnoreFile()
+		r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
+		r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
+	}
 	var cache *hosterrorscache.Cache
 	if r.options.MaxHostError > 0 {
 		cache = hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount).SetVerbose(r.options.Verbose)

v2/pkg/catalog/loader/loader.go

@@ -141,6 +141,10 @@ func (store *Store) Load() {
 // ValidateTemplates takes a list of templates and validates them
 // erroring out on discovering any faulty templates.
 func (store *Store) ValidateTemplates(templatesList, workflowsList []string) error {
+	// consider all the templates by default if no templates passed by user
+	if len(templatesList) == 0 {
+		templatesList = store.finalTemplates
+	}
 	templatePaths := store.config.Catalog.GetTemplatesPath(templatesList)
 	workflowPaths := store.config.Catalog.GetTemplatesPath(workflowsList)
 

v2/pkg/operators/matchers/match.go

@@ -3,6 +3,9 @@ package matchers
 import (
 	"strings"
 
+	"github.com/Knetic/govaluate"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
 )
 
@@ -39,7 +42,7 @@ func (matcher *Matcher) MatchSize(length int) bool {
 }
 
 // MatchWords matches a word check against a corpus.
-func (matcher *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) {
+func (matcher *Matcher) MatchWords(corpus string, data map[string]interface{}) (bool, []string) {
 	if matcher.CaseInsensitive {
 		corpus = strings.ToLower(corpus)
 	}
@@ -47,12 +50,12 @@ func (matcher *Matcher) MatchWords(corpus string, dynamicValues map[string]inter
 	var matchedWords []string
 	// Iterate over all the words accepted as valid
 	for i, word := range matcher.Words {
-		if dynamicValues == nil {
-			dynamicValues = make(map[string]interface{})
+		if data == nil {
+			data = make(map[string]interface{})
 		}
 
 		var err error
-		word, err = expressions.Evaluate(word, dynamicValues)
+		word, err = expressions.Evaluate(word, data)
 		if err != nil {
 			continue
 		}
@@ -148,6 +151,18 @@ func (matcher *Matcher) MatchBinary(corpus string) (bool, []string) {
 func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool {
 	// Iterate over all the expressions accepted as valid
 	for i, expression := range matcher.dslCompiled {
+		if varErr := expressions.ContainsUnresolvedVariables(expression.String()); varErr != nil {
+			resolvedExpression, err := expressions.Evaluate(expression.String(), data)
+			if err != nil {
+				gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcher.Name, err.Error())
+				return false
+			}
+			expression, err = govaluate.NewEvaluableExpressionWithFunctions(resolvedExpression, dsl.HelperFunctions())
+			if err != nil {
+				gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcher.Name, err.Error())
+				return false
+			}
+		}
 		result, err := expression.Evaluate(data)
 		if err != nil {
 			continue

v2/pkg/operators/matchers/match_test.go

@@ -3,6 +3,8 @@ package matchers
 import (
 	"testing"
 
+	"github.com/Knetic/govaluate"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
 	"github.com/stretchr/testify/require"
 )
 
@@ -71,3 +73,19 @@ func TestHexEncoding(t *testing.T) {
 	require.True(t, isMatched, "Could not match valid Hex condition")
 	require.Equal(t, m.Words, matched)
 }
+
+func TestMatcher_MatchDSL(t *testing.T) {
+	compiled, err := govaluate.NewEvaluableExpressionWithFunctions("contains(body, \"{{VARIABLE}}\")", dsl.HelperFunctions())
+	require.Nil(t, err, "couldn't compile expression")
+
+	m := &Matcher{Type: MatcherTypeHolder{MatcherType: DSLMatcher}, dslCompiled: []*govaluate.EvaluableExpression{compiled}}
+	err = m.CompileMatchers()
+	require.Nil(t, err, "could not compile matcher")
+
+	values := []string{"PING", "pong"}
+
+	for value := range values {
+		isMatched := m.MatchDSL(map[string]interface{}{"body": value, "VARIABLE": value})
+		require.True(t, isMatched)
+	}
+}

v2/pkg/protocols/http/http.go

@@ -123,7 +123,6 @@ type Request struct {
 	generator     *generators.PayloadGenerator // optional, only enabled when using payloads
 	httpClient    *retryablehttp.Client
 	rawhttpClient *rawhttp.Client
-	dynamicValues map[string]interface{}
 
 	// description: |
 	//   SelfContained specifies if the request is self-contained.

v2/pkg/protocols/http/operators.go

@@ -32,7 +32,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
 	case matchers.SizeMatcher:
 		return matcher.Result(matcher.MatchSize(len(item))), []string{}
 	case matchers.WordsMatcher:
-		return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, request.dynamicValues))
+		return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, data))
 	case matchers.RegexMatcher:
 		return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item))
 	case matchers.BinaryMatcher:

v2/pkg/protocols/http/request.go

@@ -249,8 +249,6 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
 			if reqURL == "" {
 				reqURL = generatedHttpRequest.URL()
 			}
-			request.dynamicValues = generatedHttpRequest.dynamicValues
-
 			// Check if hosts keep erroring
 			if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) {
 				return true, nil
@@ -520,7 +518,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 			}
 		}
 
-		event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
+		event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(generatedRequest.dynamicValues, finalEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
 			internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
 		})
 		if hasInteractMarkers {

v2/pkg/protocols/http/request_test.go

@@ -48,7 +48,7 @@ func TestHTTPExtractMultipleReuse(t *testing.T) {
 			Extractors: []*extractors.Extractor{{
 				Part:     "body",
 				Name:     "endpoint",
-				Type:     extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
+				Type:     extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor},
 				Regex:    []string{"(?m)/([a-zA-Z0-9-_/\\\\]+)"},
 				Internal: true,
 			}},

v2/pkg/protocols/network/network.go

@@ -72,9 +72,8 @@ type Request struct {
 
 	generator *generators.PayloadGenerator
 	// cache any variables that may be needed for operation.
-	dialer        *fastdialer.Dialer
-	options       *protocols.ExecuterOptions
-	dynamicValues map[string]interface{}
+	dialer  *fastdialer.Dialer
+	options *protocols.ExecuterOptions
 }
 
 // RequestPartDefinitions contains a mapping of request part definitions and their

v2/pkg/protocols/network/operators.go

@@ -23,7 +23,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
 	case matchers.SizeMatcher:
 		return matcher.Result(matcher.MatchSize(len(itemStr))), []string{}
 	case matchers.WordsMatcher:
-		return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, request.dynamicValues))
+		return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, data))
 	case matchers.RegexMatcher:
 		return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr))
 	case matchers.BinaryMatcher:

v2/pkg/protocols/network/request.go

@@ -70,6 +70,8 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 	}
 
 	payloads := generators.BuildPayloadFromOptions(request.options.Options)
+	// add Hostname variable to the payload
+	payloads = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address})
 
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
@@ -100,8 +102,6 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 		err      error
 	)
 
-	request.dynamicValues = generators.MergeMaps(payloads, variables)
-
 	if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil {
 		hostname = host
 	}
@@ -259,7 +259,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 
 	var event *output.InternalWrappedEvent
 	if len(interactshURLs) == 0 {
-		event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
+		event = eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(payloads, outputEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
 			wrappedEvent.OperatorsResult.PayloadValues = payloads
 		})
 		callback(event)