From 7e9272776d25de1963f8a962f13cb97a391df4fb Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Wed, 24 Nov 2021 22:19:42 +0530 Subject: [PATCH 1/4] add variable support to dsl, remove dynamicValues from request struct --- v2/pkg/operators/matchers/match.go | 23 +++++++++++++++++++---- v2/pkg/protocols/http/http.go | 1 - v2/pkg/protocols/http/operators.go | 2 +- v2/pkg/protocols/http/request.go | 4 ++-- v2/pkg/protocols/network/network.go | 5 ++--- v2/pkg/protocols/network/operators.go | 2 +- v2/pkg/protocols/network/request.go | 5 ++--- 7 files changed, 27 insertions(+), 15 deletions(-) diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 84601be6..d04a6149 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -3,6 +3,9 @@ package matchers import ( "strings" + "github.com/Knetic/govaluate" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" ) @@ -39,7 +42,7 @@ func (m *Matcher) MatchSize(length int) bool { } // MatchWords matches a word check against a corpus. -func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) { +func (m *Matcher) MatchWords(corpus string, data map[string]interface{}) (bool, []string) { if m.CaseInsensitive { corpus = strings.ToLower(corpus) } @@ -47,12 +50,12 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{} var matchedWords []string // Iterate over all the words accepted as valid for i, word := range m.Words { - if dynamicValues == nil { - dynamicValues = make(map[string]interface{}) + if data == nil { + data = make(map[string]interface{}) } var err error - word, err = expressions.Evaluate(word, dynamicValues) + word, err = expressions.Evaluate(word, data) if err != nil { continue } @@ -148,6 +151,18 @@ func (m *Matcher) MatchBinary(corpus string) (bool, []string) { func (m *Matcher) MatchDSL(data map[string]interface{}) bool { // Iterate over all the expressions accepted as valid for i, expression := range m.dslCompiled { + if varErr := expressions.ContainsUnresolvedVariables(expression.String()); varErr != nil { + resolvedExpression, err := expressions.Evaluate(expression.String(), data) + if err != nil { + gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", m.Name, err.Error()) + return false + } + expression, err = govaluate.NewEvaluableExpressionWithFunctions(resolvedExpression, dsl.HelperFunctions()) + if err != nil { + gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", m.Name, err.Error()) + return false + } + } result, err := expression.Evaluate(data) if err != nil { continue diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 17a87588..49e7c74d 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -134,7 +134,6 @@ type Request struct { generator *generators.PayloadGenerator // optional, only enabled when using payloads httpClient *retryablehttp.Client rawhttpClient *rawhttp.Client - dynamicValues map[string]interface{} // description: | // SelfContained specifies if the request is self contained. diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 1f319ccf..70fd4d4c 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -32,7 +32,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: - return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, request.dynamicValues)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, data)) case matchers.RegexMatcher: return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item)) case matchers.BinaryMatcher: diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 8f204078..405e45bf 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -231,7 +231,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou if reqURL == "" { reqURL = generatedHttpRequest.URL() } - request.dynamicValues = generatedHttpRequest.dynamicValues + // Check if hosts just keep erroring if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) { break @@ -470,7 +470,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } } - event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(generatedRequest.dynamicValues, finalEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta }) if hasInteractMarkers { diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 09dcfb45..385b5bb7 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -77,9 +77,8 @@ type Request struct { generator *generators.PayloadGenerator // cache any variables that may be needed for operation. - dialer *fastdialer.Dialer - options *protocols.ExecuterOptions - dynamicValues map[string]interface{} + dialer *fastdialer.Dialer + options *protocols.ExecuterOptions } type addressKV struct { diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 010b11d6..d3083301 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -23,7 +23,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, request.dynamicValues)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, data)) case matchers.RegexMatcher: return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 35da81d1..484f95ae 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -78,6 +78,7 @@ func (request *Request) executeAddress(actualAddress, address, input string, sho } payloads := generators.BuildPayloadFromOptions(request.options.Options) + generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) if request.generator != nil { iterator := request.generator.NewIterator() @@ -108,8 +109,6 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input err error ) - request.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) - if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil { hostname = host } @@ -267,7 +266,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input var event *output.InternalWrappedEvent if len(interactshURLs) == 0 { - event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { + event = eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(payloads, outputEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { wrappedEvent.OperatorsResult.PayloadValues = payloads }) callback(event) From 5deb454a81f4e32639d6f2e2ad606ba425499130 Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Mon, 29 Nov 2021 13:26:03 +0530 Subject: [PATCH 2/4] merge Hostname variable to payloads in network request --- v2/pkg/protocols/network/request.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 484f95ae..24d8b260 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -78,7 +78,8 @@ func (request *Request) executeAddress(actualAddress, address, input string, sho } payloads := generators.BuildPayloadFromOptions(request.options.Options) - generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) + // add Hostname variable to the payload + payloads = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) if request.generator != nil { iterator := request.generator.NewIterator() From 5b99921d75b48f30d22f8ead6d459f6e58337a7f Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Tue, 30 Nov 2021 20:20:43 +0530 Subject: [PATCH 3/4] add unit and integration tests for dsl variable #555 --- .../http/dsl-matcher-variable.yaml | 23 +++++++++++++++++++ v2/cmd/integration-test/http.go | 22 ++++++++++++++++++ v2/pkg/operators/matchers/match_test.go | 18 +++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 integration_tests/http/dsl-matcher-variable.yaml diff --git a/integration_tests/http/dsl-matcher-variable.yaml b/integration_tests/http/dsl-matcher-variable.yaml new file mode 100644 index 00000000..ecbe5f9e --- /dev/null +++ b/integration_tests/http/dsl-matcher-variable.yaml @@ -0,0 +1,23 @@ +id: dsl-matcher-variable + +info: + name: dsl-matcher-variable + author: pd-team + severity: info + +requests: + - + path: + - "{{BaseURL}}" + payloads: + VALUES: + - This + - is + - test + - matcher + - text + matchers: + - + dsl: + - 'contains(body,"{{VALUES}}")' + type: dsl \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 1890048d..ebf24531 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -36,6 +36,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{}, "http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{}, "http/get-redirects-chain-headers.yaml": &httpGetRedirectsChainHeaders{}, + "http/dsl-matcher-variable.yaml": &httpDSLVariable{}, } type httpInteractshRequest struct{} @@ -155,6 +156,27 @@ func (h *httpGet) Execute(filePath string) error { return nil } +type httpDSLVariable struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpDSLVariable) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "This is test matcher text") + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if len(results) != 5 { + return errIncorrectResultsCount(results) + } + return nil +} + type httpPostBody struct{} // Execute executes a test case and returns an error if occurred diff --git a/v2/pkg/operators/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go index b52adbc6..68a6d1b0 100644 --- a/v2/pkg/operators/matchers/match_test.go +++ b/v2/pkg/operators/matchers/match_test.go @@ -3,6 +3,8 @@ package matchers import ( "testing" + "github.com/Knetic/govaluate" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/stretchr/testify/require" ) @@ -71,3 +73,19 @@ func TestHexEncoding(t *testing.T) { require.True(t, isMatched, "Could not match valid Hex condition") require.Equal(t, m.Words, matched) } + +func TestMatcher_MatchDSL(t *testing.T) { + compiled, err := govaluate.NewEvaluableExpressionWithFunctions("contains(body, \"{{VARIABLE}}\")", dsl.HelperFunctions()) + require.Nil(t, err, "couldn't compile expression") + + m := &Matcher{Type: MatcherTypeHolder{MatcherType: DSLMatcher}, dslCompiled: []*govaluate.EvaluableExpression{compiled}} + err = m.CompileMatchers() + require.Nil(t, err, "could not compile matcher") + + values := []string{"PING", "pong"} + + for value := range values { + isMatched := m.MatchDSL(map[string]interface{}{"body": value, "VARIABLE": value}) + require.True(t, isMatched) + } +} From 6d5146e540665ea59f0b748c5c9fbde9ddc4bee1 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Wed, 1 Dec 2021 10:35:18 -0600 Subject: [PATCH 4/4] validate flag updates --- v2/internal/runner/options.go | 2 +- v2/internal/runner/runner.go | 12 ++++++++---- v2/pkg/catalog/loader/loader.go | 4 ++++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index f11d2055..de50701d 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -116,7 +116,7 @@ func validateOptions(options *types.Options) error { // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output - if options.Verbose { + if options.Verbose || options.Validate { gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) } if options.Debug { diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 055eca1a..bb764942 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -69,6 +69,8 @@ func New(options *types.Options) (*Runner, error) { } if options.Validate { parsers.ShouldValidate = true + // Does not update the templates when validate flag is used + options.NoUpdateTemplates = true } if err := runner.updateTemplates(); err != nil { gologger.Warning().Msgf("Could not update templates: %s\n", err) @@ -231,10 +233,12 @@ func (r *Runner) RunEnumeration() error { } r.options.Templates = append(r.options.Templates, templatesLoaded...) } - ignoreFile := config.ReadIgnoreFile() - r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...) - r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...) - + // Exclude ignored file for validation + if !r.options.Validate { + ignoreFile := config.ReadIgnoreFile() + r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...) + r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...) + } var cache *hosterrorscache.Cache if r.options.MaxHostError > 0 { cache = hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount).SetVerbose(r.options.Verbose) diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 1a61cb25..a1e3b474 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -141,6 +141,10 @@ func (store *Store) Load() { // ValidateTemplates takes a list of templates and validates them // erroring out on discovering any faulty templates. func (store *Store) ValidateTemplates(templatesList, workflowsList []string) error { + // consider all the templates by default if no templates passed by user + if len(templatesList) == 0 { + templatesList = store.finalTemplates + } templatePaths := store.config.Catalog.GetTemplatesPath(templatesList) workflowPaths := store.config.Catalog.GetTemplatesPath(workflowsList)