diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 6d43c0cc..28b52d7c 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -7,7 +7,6 @@ import ( "net" "net/http" "net/http/httptest" - "net/http/httputil" "strings" "github.com/julienschmidt/httprouter" @@ -34,21 +33,12 @@ var httpTestcases = map[string]testutils.TestCase{ "http/request-condition-new.yaml": &httpRequestCondition{}, } -func httpDebugRequestDump(r *http.Request) { - if debug { - if dump, err := httputil.DumpRequest(r, true); err == nil { - fmt.Printf("\nRequest dump: \n%s\n\n", string(dump)) - } - } -} - type httpGetHeaders struct{} // Execute executes a test case and returns an error if occurred func (h *httpGetHeaders) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if strings.EqualFold(r.Header.Get("test"), "nuclei") { fmt.Fprintf(w, "This is test headers matcher text") } @@ -72,7 +62,6 @@ type httpGetQueryString struct{} func (h *httpGetQueryString) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") { fmt.Fprintf(w, "This is test querystring matcher text") } @@ -96,11 +85,9 @@ type httpGetRedirects struct{} func (h *httpGetRedirects) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) http.Redirect(w, r, "/redirected", http.StatusFound) }) router.GET("/redirected", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test redirects matcher text") }) ts := httptest.NewServer(router) @@ -122,7 +109,6 @@ type httpGet struct{} func (h *httpGet) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") }) ts := httptest.NewServer(router) @@ -146,7 +132,6 @@ func (h *httpPostBody) Execute(filePath string) error { var routerErr error router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err return @@ -179,8 +164,6 @@ func (h *httpPostJSONBody) Execute(filePath string) error { var routerErr error router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) - type doc struct { Username string `json:"username"` Password string `json:"password"` @@ -218,7 +201,6 @@ func (h *httpPostMultipartBody) Execute(filePath string) error { var routerErr error router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if err := r.ParseMultipartForm(1 * 1024); err != nil { routerErr = err return @@ -261,7 +243,6 @@ func (h *httpRawDynamicExtractor) Execute(filePath string) error { var routerErr error router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err return @@ -271,7 +252,6 @@ func (h *httpRawDynamicExtractor) Execute(filePath string) error { } }) router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if strings.EqualFold(r.URL.Query().Get("username"), "nuclei") { fmt.Fprintf(w, "Test is test-dynamic-extractor-raw matcher text") } @@ -300,7 +280,6 @@ func (h *httpRawGetQuery) Execute(filePath string) error { var routerErr error router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") { fmt.Fprintf(w, "Test is test raw-get-query-matcher text") } @@ -329,8 +308,6 @@ func (h *httpRawGet) Execute(filePath string) error { var routerErr error router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) - fmt.Fprintf(w, "Test is test raw-get-matcher text") }) ts := httptest.NewServer(router) @@ -357,7 +334,6 @@ func (h *httpRawPayload) Execute(filePath string) error { var routerErr error router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err return @@ -393,7 +369,6 @@ func (h *httpRawPostBody) Execute(filePath string) error { var routerErr error router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err return @@ -426,7 +401,6 @@ func (h *httpRawCookieReuse) Execute(filePath string) error { var routerErr error router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err return @@ -436,7 +410,6 @@ func (h *httpRawCookieReuse) Execute(filePath string) error { } }) router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err return @@ -500,11 +473,9 @@ func (h *httpRequestCondition) Execute(filePath string) error { var routerErr error router.GET("/200", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) w.WriteHeader(200) }) router.GET("/400", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) w.WriteHeader(400) }) ts := httptest.NewServer(router) diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go index a377a1c0..8c738537 100644 --- a/v2/cmd/integration-test/network.go +++ b/v2/cmd/integration-test/network.go @@ -82,7 +82,14 @@ func (h *networkMultiStep) Execute(filePath string) error { if routerErr != nil { return routerErr } - if len(results) != 1 { + + var expectedResultsSize int + if debug { + expectedResultsSize = 3 + } else { + expectedResultsSize = 1 + } + if len(results) != expectedResultsSize { return errIncorrectResultsCount(results) } return nil diff --git a/v2/cmd/integration-test/workflow.go b/v2/cmd/integration-test/workflow.go index 08d6e138..5f39b4eb 100644 --- a/v2/cmd/integration-test/workflow.go +++ b/v2/cmd/integration-test/workflow.go @@ -23,7 +23,6 @@ type workflowBasic struct{} func (h *workflowBasic) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") }) ts := httptest.NewServer(router) @@ -45,7 +44,6 @@ type workflowConditionMatched struct{} func (h *workflowConditionMatched) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") }) ts := httptest.NewServer(router) @@ -67,7 +65,6 @@ type workflowConditionUnmatch struct{} func (h *workflowConditionUnmatch) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") }) ts := httptest.NewServer(router) @@ -89,7 +86,6 @@ type workflowMatcherName struct{} func (h *workflowMatcherName) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") }) ts := httptest.NewServer(router) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 2b4cc06d..217f3400 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -48,7 +48,7 @@ var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`) // // If the path exists but does not contain the latest version of public templates, // the new version is downloaded from GitHub to the templates' directory, overwriting the old content. -func (r *Runner) updateTemplates() error { +func (r *Runner) updateTemplates() error { // TODO this method does more than just update templates. Should be refactored. home, err := os.UserHomeDir() if err != nil { return err @@ -75,6 +75,7 @@ func (r *Runner) updateTemplates() error { if r.options.NoUpdateTemplates && !r.options.UpdateTemplates { return nil } + client.InitNucleiVersion(config.Version) r.fetchLatestVersionsFromGithub(configDir) // also fetch the latest versions @@ -121,59 +122,74 @@ func (r *Runner) updateTemplates() error { return nil } - // Get the configuration currently on disk. - verText := r.templatesConfig.TemplateVersion - indices := reVersion.FindStringIndex(verText) - if indices == nil { - return fmt.Errorf("invalid release found with tag %s", err) - } - if indices[0] > 0 { - verText = verText[indices[0]:] - } - - oldVersion, err := semver.Make(verText) + latestVersion, currentVersion, err := getVersions(r) if err != nil { return err } - version, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion) - if err != nil { - return err - } - - if version.EQ(oldVersion) { + if latestVersion.EQ(currentVersion) { if r.options.UpdateTemplates { gologger.Info().Msgf("No new updates found for nuclei templates") } return config.WriteConfiguration(r.templatesConfig) } - if version.GT(oldVersion) { - gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String()) + if err := updateTemplates(latestVersion, currentVersion, r, ctx); err != nil { + return err + } + return nil +} + +func updateTemplates(latestVersion semver.Version, currentVersion semver.Version, runner *Runner, ctx context.Context) error { + if latestVersion.GT(currentVersion) { + gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", currentVersion, latestVersion.String()) gologger.Info().Msgf("Downloading latest release...") - if r.options.TemplatesDirectory != "" { - r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory + if runner.options.TemplatesDirectory != "" { + runner.templatesConfig.TemplatesDirectory = runner.options.TemplatesDirectory } - r.templatesConfig.TemplateVersion = version.String() + runner.templatesConfig.TemplateVersion = latestVersion.String() - gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) + gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", latestVersion.String(), runner.templatesConfig.TemplatesDirectory) - asset, err := r.getLatestReleaseFromGithub(r.templatesConfig.NucleiTemplatesLatestVersion) + asset, err := runner.getLatestReleaseFromGithub(runner.templatesConfig.NucleiTemplatesLatestVersion) if err != nil { return err } - if _, err := r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()); err != nil { + if _, err := runner.downloadReleaseAndUnzip(ctx, latestVersion.String(), asset.GetZipballURL()); err != nil { return err } - if err := config.WriteConfiguration(r.templatesConfig); err != nil { + if err := config.WriteConfiguration(runner.templatesConfig); err != nil { return err } - gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", version.String()) + gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", latestVersion.String()) } return nil } +func getVersions(runner *Runner) (semver.Version, semver.Version, error) { + // Get the configuration currently on disk. + verText := runner.templatesConfig.TemplateVersion + indices := reVersion.FindStringIndex(verText) + if indices == nil { + return semver.Version{}, semver.Version{}, fmt.Errorf("invalid release found with tag %s", verText) + } + if indices[0] > 0 { + verText = verText[indices[0]:] + } + + currentVersion, err := semver.Make(verText) + if err != nil { + return semver.Version{}, semver.Version{}, err + } + + latestVersion, err := semver.Parse(runner.templatesConfig.NucleiTemplatesLatestVersion) + if err != nil { + return semver.Version{}, semver.Version{}, err + } + return latestVersion, currentVersion, nil +} + // readInternalConfigurationFile reads the internal configuration file for nuclei func (r *Runner) readInternalConfigurationFile(home, configDir string) error { templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename) @@ -210,9 +226,9 @@ func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool { // getLatestReleaseFromGithub returns the latest release from GitHub func (r *Runner) getLatestReleaseFromGithub(latestTag string) (*github.RepositoryRelease, error) { - client := github.NewClient(nil) + gitHubClient := github.NewClient(nil) - release, _, err := client.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag) + release, _, err := gitHubClient.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag) if err != nil { return nil, err } @@ -361,7 +377,7 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat for templatePath, templateChecksums := range templateChecksumsMap { _, ok := results.checksums[templatePath] if !ok && templateChecksums[0] == templateChecksums[1] { - os.Remove(templatePath) + _ = os.Remove(templatePath) results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(templatePath, r.templatesConfig.TemplatesDirectory), string(os.PathSeparator))) } } @@ -456,7 +472,7 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st // fetchLatestVersionsFromGithub fetches the latest versions of nuclei repos from GitHub // -// This fetches latest nuclei/templates/ignore from https://version-check.nuclei.sh/versions +// This fetches the latest nuclei/templates/ignore from https://version-check.nuclei.sh/versions // If you want to disable this automatic update check, use -nut flag. func (r *Runner) fetchLatestVersionsFromGithub(configDir string) { versions, err := client.GetLatestNucleiTemplatesVersion() diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 2eb49569..41ce5361 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -6,8 +6,9 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" - "github.com/projectdiscovery/gologger" "gopkg.in/yaml.v2" + + "github.com/projectdiscovery/gologger" ) // Config contains the internal nuclei engine configuration diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 401110cf..fff0055d 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "strings" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" ) @@ -40,7 +41,8 @@ func (m *Matcher) MatchSize(length int) bool { } // MatchWords matches a word check against a corpus. -func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) bool { +func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) { + var matchedWords []string // Iterate over all the words accepted as valid for i, word := range m.Words { if dynamicValues == nil { @@ -57,7 +59,7 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{} // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. if m.condition == ANDCondition { - return false + return false, []string{} } // Continue with the flow since it's an OR Condition. continue @@ -65,19 +67,22 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{} // If the condition was an OR, return on the first match. if m.condition == ORCondition { - return true + return true, []string{word} } + matchedWords = append(matchedWords, word) + // If we are at the end of the words, return with true if len(m.Words)-1 == i { - return true + return true, matchedWords } } - return false + return false, []string{} } // MatchRegex matches a regex check against a corpus -func (m *Matcher) MatchRegex(corpus string) bool { +func (m *Matcher) MatchRegex(corpus string) (bool, []string) { + var matchedRegexes []string // Iterate over all the regexes accepted as valid for i, regex := range m.regexCompiled { // Continue if the regex doesn't match @@ -85,36 +90,47 @@ func (m *Matcher) MatchRegex(corpus string) bool { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. if m.condition == ANDCondition { - return false + return false, []string{} } // Continue with the flow since it's an OR Condition. continue } + currentMatches := regex.FindAllString(corpus, -1) // If the condition was an OR, return on the first match. if m.condition == ORCondition { - return true + return true, currentMatches } + matchedRegexes = append(matchedRegexes, currentMatches...) + // If we are at the end of the regex, return with true if len(m.regexCompiled)-1 == i { - return true + return true, matchedRegexes } } - return false + return false, []string{} } // MatchBinary matches a binary check against a corpus -func (m *Matcher) MatchBinary(corpus string) bool { +func (m *Matcher) MatchBinary(corpus string) (bool, []string) { + var matchedBinary []string // Iterate over all the words accepted as valid for i, binary := range m.Binary { // Continue if the word doesn't match - hexa, _ := hex.DecodeString(binary) + hexa, err := hex.DecodeString(binary) + if err != nil { + gologger.Warning().Msgf("Could not hex encode the given binary matcher value: '%s'", binary) + if m.condition == ANDCondition { + return false, []string{} + } + continue + } if !strings.Contains(corpus, string(hexa)) { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. if m.condition == ANDCondition { - return false + return false, []string{} } // Continue with the flow since it's an OR Condition. continue @@ -122,15 +138,17 @@ func (m *Matcher) MatchBinary(corpus string) bool { // If the condition was an OR, return on the first match. if m.condition == ORCondition { - return true + return true, []string{string(hexa)} } + matchedBinary = append(matchedBinary, string(hexa)) + // If we are at the end of the words, return with true if len(m.Binary)-1 == i { - return true + return true, matchedBinary } } - return false + return false, []string{} } // MatchDSL matches on a generic map result diff --git a/v2/pkg/operators/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go index 6a2d0b85..4c2f9bc0 100644 --- a/v2/pkg/operators/matchers/match_test.go +++ b/v2/pkg/operators/matchers/match_test.go @@ -6,27 +6,60 @@ import ( "github.com/stretchr/testify/require" ) -func TestANDCondition(t *testing.T) { +func TestWordANDCondition(t *testing.T) { m := &Matcher{condition: ANDCondition, Words: []string{"a", "b"}} - matched := m.MatchWords("a b", nil) - require.True(t, matched, "Could not match valid AND condition") + isMatched, matched := m.MatchWords("a b", nil) + require.True(t, isMatched, "Could not match words with valid AND condition") + require.Equal(t, m.Words, matched) - matched = m.MatchWords("b", nil) - require.False(t, matched, "Could match invalid AND condition") + isMatched, matched = m.MatchWords("b", nil) + require.False(t, isMatched, "Could match words with invalid AND condition") + require.Equal(t, []string{}, matched) +} + +func TestRegexANDCondition(t *testing.T) { + m := &Matcher{Type: "regex", Condition: "and", Regex: []string{"[a-z]{3}", "\\d{2}"}} + err := m.CompileMatchers() + require.Nil(t, err) + + isMatched, matched := m.MatchRegex("abc abcd 123") + require.True(t, isMatched, "Could not match regex with valid AND condition") + require.Equal(t, []string{"abc", "abc", "12"}, matched) + + isMatched, matched = m.MatchRegex("bc 1") + require.False(t, isMatched, "Could match regex with invalid AND condition") + require.Equal(t, []string{}, matched) } func TestORCondition(t *testing.T) { m := &Matcher{condition: ORCondition, Words: []string{"a", "b"}} - matched := m.MatchWords("a b", nil) - require.True(t, matched, "Could not match valid OR condition") + isMatched, matched := m.MatchWords("a b", nil) + require.True(t, isMatched, "Could not match valid word OR condition") + require.Equal(t, []string{"a"}, matched) - matched = m.MatchWords("b", nil) - require.True(t, matched, "Could not match valid OR condition") + isMatched, matched = m.MatchWords("b", nil) + require.True(t, isMatched, "Could not match valid word OR condition") + require.Equal(t, []string{"b"}, matched) - matched = m.MatchWords("c", nil) - require.False(t, matched, "Could match invalid OR condition") + isMatched, matched = m.MatchWords("c", nil) + require.False(t, isMatched, "Could match invalid word OR condition") + require.Equal(t, []string{}, matched) +} + +func TestRegexOrCondition(t *testing.T) { + m := &Matcher{Type: "regex", Condition: "or", Regex: []string{"[a-z]{3}", "\\d{2}"}} + err := m.CompileMatchers() + require.Nil(t, err) + + isMatched, matched := m.MatchRegex("ab 123") + require.True(t, isMatched, "Could not match valid regex OR condition") + require.Equal(t, []string{"12"}, matched) + + isMatched, matched = m.MatchRegex("bc 1") + require.False(t, isMatched, "Could match invalid regex OR condition") + require.Equal(t, []string{}, matched) } func TestHexEncoding(t *testing.T) { @@ -34,6 +67,7 @@ func TestHexEncoding(t *testing.T) { err := m.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := m.MatchWords("PING", nil) - require.True(t, matched, "Could not match valid Hex condition") + isMatched, matched := m.MatchWords("PING", nil) + require.True(t, isMatched, "Could not match valid Hex condition") + require.Equal(t, m.Words, matched) } diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index a0d17647..88ee413e 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -165,6 +165,14 @@ func (m *Matcher) Result(data bool) bool { return data } +// ResultWithMatchedSnippet returns true and the matched snippet, or false and an empty string +func (m *Matcher) ResultWithMatchedSnippet(data bool, matchedSnippet []string) (bool, []string) { + if m.Negative { + return !data, []string{} + } + return data, matchedSnippet +} + // GetType returns the type of the matcher func (m *Matcher) GetType() MatcherType { return m.matcherType diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 493eddf0..af13b4d9 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -1,6 +1,8 @@ package operators import ( + "strconv" + "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" @@ -32,19 +34,19 @@ type Operators struct { } // Compile compiles the operators as well as their corresponding matchers and extractors -func (r *Operators) Compile() error { - if r.MatchersCondition != "" { - r.matchersCondition = matchers.ConditionTypes[r.MatchersCondition] +func (operators *Operators) Compile() error { + if operators.MatchersCondition != "" { + operators.matchersCondition = matchers.ConditionTypes[operators.MatchersCondition] } else { - r.matchersCondition = matchers.ORCondition + operators.matchersCondition = matchers.ORCondition } - for _, matcher := range r.Matchers { + for _, matcher := range operators.Matchers { if err := matcher.CompileMatchers(); err != nil { return errors.Wrap(err, "could not compile matcher") } } - for _, extractor := range r.Extractors { + for _, extractor := range operators.Extractors { if err := extractor.CompileExtractors(); err != nil { return errors.Wrap(err, "could not compile extractor") } @@ -53,8 +55,8 @@ func (r *Operators) Compile() error { } // GetMatchersCondition returns the condition for the matchers -func (r *Operators) GetMatchersCondition() matchers.ConditionType { - return r.matchersCondition +func (operators *Operators) GetMatchersCondition() matchers.ConditionType { + return operators.matchersCondition } // Result is a result structure created from operators running on data. @@ -64,7 +66,7 @@ type Result struct { // Extracted is true if any result type values were extracted Extracted bool // Matches is a map of matcher names that we matched - Matches map[string]struct{} + Matches map[string][]string // Extracts contains all the data extracted from inputs Extracts map[string][]string // OutputExtracts is the list of extracts to be displayed on screen. @@ -100,24 +102,24 @@ func (r *Result) Merge(result *Result) { } // MatchFunc performs matching operation for a matcher on model and returns true or false. -type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) bool +type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) // ExtractFunc performs extracting operation for an extractor on model and returns true or false. type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // Execute executes the operators on data and returns a result structure -func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc) (*Result, bool) { - matcherCondition := r.GetMatchersCondition() +func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc, isDebug bool) (*Result, bool) { + matcherCondition := operators.GetMatchersCondition() var matches bool result := &Result{ - Matches: make(map[string]struct{}), + Matches: make(map[string][]string), Extracts: make(map[string][]string), DynamicValues: make(map[string]interface{}), } // Start with the extractors first and evaluate them. - for _, extractor := range r.Extractors { + for _, extractor := range operators.Extractors { var extractorResults []string for match := range extract(data, extractor) { @@ -136,23 +138,22 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac } } - for _, matcher := range r.Matchers { - // Check if the matcher matched - if !match(data, matcher) { - // If the condition is AND we haven't matched, try next request. - if matcherCondition == matchers.ANDCondition { - if len(result.DynamicValues) > 0 { - return result, true + for matcherIndex, matcher := range operators.Matchers { + if isMatch, matched := match(data, matcher); isMatch { + if isDebug { // matchers without an explicit name or with AND condition should only be made visible if debug is enabled + matcherName := getMatcherName(matcher, matcherIndex) + result.Matches[matcherName] = matched + } else { // if it's a "named" matcher with OR condition, then display it + if matcherCondition == matchers.ORCondition && matcher.Name != "" { + result.Matches[matcher.Name] = matched } - return nil, false - } - } else { - // If the matcher has matched, and it's an OR - // write the first output then move to next matcher. - if matcherCondition == matchers.ORCondition && matcher.Name != "" { - result.Matches[matcher.Name] = struct{}{} } matches = true + } else if matcherCondition == matchers.ANDCondition { + if len(result.DynamicValues) > 0 { + return result, true + } + return nil, false } } @@ -162,8 +163,8 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac return result, true } - // Don't print if we have matchers and they have not matched, regardless of extractor - if len(r.Matchers) > 0 && !matches { + // Don't print if we have matchers, and they have not matched, regardless of extractor + if len(operators.Matchers) > 0 && !matches { return nil, false } // Write a final string of output if matcher type is @@ -174,12 +175,20 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac return nil, false } +func getMatcherName(matcher *matchers.Matcher, matcherIndex int) string { + if matcher.Name != "" { + return matcher.Name + } else { + return matcher.Type + "-" + strconv.Itoa(matcherIndex+1) // making the index start from 1 to be more readable + } +} + // ExecuteInternalExtractors executes internal dynamic extractors -func (r *Operators) ExecuteInternalExtractors(data map[string]interface{}, extract ExtractFunc) map[string]interface{} { +func (operators *Operators) ExecuteInternalExtractors(data map[string]interface{}, extract ExtractFunc) map[string]interface{} { dynamicValues := make(map[string]interface{}) // Start with the extractors first and evaluate them. - for _, extractor := range r.Extractors { + for _, extractor := range operators.Extractors { if !extractor.Internal { continue } diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/protocols/common/clusterer/executer.go index ac523c8d..c060f341 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/protocols/common/clusterer/executer.go @@ -67,7 +67,7 @@ func (e *Executer) Execute(input string) (bool, error) { dynamicValues := make(map[string]interface{}) err := e.requests.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { - result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract) + result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) if matched && result != nil { event.OperatorsResult = result event.InternalEvent["template-id"] = operator.templateID @@ -98,7 +98,7 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve dynamicValues := make(map[string]interface{}) err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { - result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract) + result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) if matched && result != nil { event.OperatorsResult = result event.InternalEvent["template-id"] = operator.templateID diff --git a/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go b/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go new file mode 100644 index 00000000..c94eb15a --- /dev/null +++ b/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go @@ -0,0 +1,29 @@ +package eventcreator + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" +) + +// CreateEvent wraps the outputEvent with the result of the operators defined on the request +func CreateEvent(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool) *output.InternalWrappedEvent { + return CreateEventWithAdditionalOptions(request, outputEvent, isResponseDebug, func(internalWrappedEvent *output.InternalWrappedEvent) {}) +} + +// CreateEventWithAdditionalOptions wraps the outputEvent with the result of the operators defined on the request and enables extending the resulting event with additional attributes or values. +func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool, + addAdditionalOptions func(internalWrappedEvent *output.InternalWrappedEvent)) *output.InternalWrappedEvent { + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + for _, compiledOperator := range request.GetCompiledOperators() { + if compiledOperator != nil { + result, ok := compiledOperator.Execute(outputEvent, request.Match, request.Extract, isResponseDebug) + if ok && result != nil { + event.OperatorsResult = result + addAdditionalOptions(event) + event.Results = append(event.Results, request.MakeResultEvent(event)...) + } + } + } + + return event +} diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go new file mode 100644 index 00000000..1cb914e9 --- /dev/null +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go @@ -0,0 +1,35 @@ +package responsehighlighter + +import ( + "strconv" + "strings" + + "github.com/logrusorgru/aurora" + + "github.com/projectdiscovery/nuclei/v2/pkg/operators" +) + +var colorizer = aurora.NewAurora(true) + +func Highlight(operatorResult *operators.Result, response string, noColor bool) string { + result := response + if operatorResult != nil && !noColor { + for _, matches := range operatorResult.Matches { + if len(matches) > 0 { + for _, currentMatch := range matches { + result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + } + } + } + } + + return result +} + +func CreateStatusCodeSnippet(response string, statusCode int) string { + if strings.HasPrefix(response, "HTTP/") { + strStatusCode := strconv.Itoa(statusCode) + return response[:strings.Index(response, strStatusCode)+len(strStatusCode)] + } + return "" +} diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 0d11c7af..5c2914ff 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -145,7 +145,7 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol data.Event.InternalEvent["interactsh_request"] = interaction.RawRequest data.Event.InternalEvent["interactsh_response"] = interaction.RawResponse - result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc) + result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc, false) if !matched || result == nil { return false // if we don't match, return } diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index e4e7833e..15f42dc0 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -76,47 +76,51 @@ type Request struct { Resolvers []string `yaml:"resolvers,omitempty" jsonschema:"title=Resolvers,description=Define resolvers to use within the template"` } +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { - return r.ID +func (request *Request) GetID() string { + return request.ID } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { +func (request *Request) Compile(options *protocols.ExecuterOptions) error { dnsClientOptions := &dnsclientpool.Configuration{ - Retries: r.Retries, + Retries: request.Retries, } - if len(r.Resolvers) > 0 { - dnsClientOptions.Resolvers = r.Resolvers + if len(request.Resolvers) > 0 { + dnsClientOptions.Resolvers = request.Resolvers } // Create a dns client for the class client, err := dnsclientpool.Get(options.Options, dnsClientOptions) if err != nil { return errors.Wrap(err, "could not get dns client") } - r.dnsClient = client + request.dnsClient = client - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } - r.class = classToInt(r.Class) - r.options = options - r.question = questionTypeToInt(r.Type) + request.class = classToInt(request.Class) + request.options = options + request.question = questionTypeToInt(request.Type) return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { +func (request *Request) Requests() int { return 1 } // Make returns the request to be sent for the protocol -func (r *Request) Make(domain string) (*dns.Msg, error) { - if r.question != dns.TypePTR && net.ParseIP(domain) != nil { +func (request *Request) Make(domain string) (*dns.Msg, error) { + if request.question != dns.TypePTR && net.ParseIP(domain) != nil { return nil, errors.New("cannot use IP address as DNS input") } domain = dns.Fqdn(domain) @@ -124,20 +128,20 @@ func (r *Request) Make(domain string) (*dns.Msg, error) { // Build a request on the specified URL req := new(dns.Msg) req.Id = dns.Id() - req.RecursionDesired = r.Recursion + req.RecursionDesired = request.Recursion var q dns.Question - final := replacer.Replace(r.Name, map[string]interface{}{"FQDN": domain}) + final := replacer.Replace(request.Name, map[string]interface{}{"FQDN": domain}) q.Name = dns.Fqdn(final) - q.Qclass = r.class - q.Qtype = r.question + q.Qclass = request.class + q.Qtype = request.question req.Question = append(req.Question, q) req.SetEdns0(4096, false) - switch r.question { + switch request.question { case dns.TypeTXT: req.AuthenticatedData = true } diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index cda725d3..b9f0454b 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -10,11 +10,12 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "all", "": @@ -23,28 +24,32 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } switch matcher.GetType() { case matchers.StatusMatcher: - return matcher.Result(matcher.MatchStatusCode(item.(int))) + statusCode, ok := item.(int) + if !ok { + return false, []string{} + } + return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{} case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(types.ToString(item)))) + return matcher.Result(matcher.MatchSize(len(types.ToString(item)))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(types.ToString(item), nil)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(types.ToString(item), nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(types.ToString(item))) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(types.ToString(item))) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(types.ToString(item))) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(types.ToString(item))) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { part := extractor.Part switch part { case "body", "all": @@ -67,77 +72,29 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext } // responseToDSLMap converts a DNS response to a map for use in DSL matching -func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent { - data := make(output.InternalEvent, 11) - - // Some data regarding the request metadata - data["host"] = host - data["matched"] = matched - data["request"] = req.String() - - data["rcode"] = resp.Rcode - buffer := &bytes.Buffer{} - for _, question := range resp.Question { - buffer.WriteString(question.String()) +func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent { + return output.InternalEvent{ + "host": host, + "matched": matched, + "request": req.String(), + "rcode": resp.Rcode, + "question": questionToString(resp.Question), + "extra": rrToString(resp.Extra), + "answer": rrToString(resp.Answer), + "ns": rrToString(resp.Ns), + "raw": resp.String(), + "template-id": request.options.TemplateID, + "template-info": request.options.TemplateInfo, + "template-path": request.options.TemplatePath, } - data["question"] = buffer.String() - buffer.Reset() - - for _, extra := range resp.Extra { - buffer.WriteString(extra.String()) - } - data["extra"] = buffer.String() - buffer.Reset() - - for _, answer := range resp.Answer { - buffer.WriteString(answer.String()) - } - data["answer"] = buffer.String() - buffer.Reset() - - for _, ns := range resp.Ns { - buffer.WriteString(ns.String()) - } - data["ns"] = buffer.String() - buffer.Reset() - - rawData := resp.String() - data["raw"] = rawData - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath - return data } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := r.makeResultEventItem(wrapped) - results = append(results, data) - } - return results +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), @@ -152,3 +109,19 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out } return data } + +func rrToString(resourceRecords []dns.RR) string { // TODO rewrite with generics when available + buffer := &bytes.Buffer{} + for _, resourceRecord := range resourceRecords { + buffer.WriteString(resourceRecord.String()) + } + return buffer.String() +} + +func questionToString(resourceRecords []dns.Question) string { + buffer := &bytes.Buffer{} + for _, resourceRecord := range resourceRecords { + buffer.WriteString(resourceRecord.String()) + } + return buffer.String() +} diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index 286dd582..c8d06124 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -87,8 +87,9 @@ func TestDNSOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatch, matched := request.Match(event, matcher) + require.True(t, isMatch, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("rcode", func(t *testing.T) { @@ -100,8 +101,9 @@ func TestDNSOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile rcode matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid rcode response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid rcode response") + require.Equal(t, []string{}, matched) }) t.Run("negative", func(t *testing.T) { @@ -114,8 +116,9 @@ func TestDNSOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -127,8 +130,9 @@ func TestDNSOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } @@ -232,13 +236,15 @@ func TestDNSMakeResult(t *testing.T) { event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one") finalEvent := &output.InternalWrappedEvent{InternalEvent: event} if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract) + result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false) if ok && result != nil { finalEvent.OperatorsResult = result finalEvent.Results = request.MakeResultEvent(finalEvent) } } require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results") - require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results") - require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results") + resultEvent := finalEvent.Results[0] + require.Equal(t, "test", resultEvent.MatcherName, "could not get correct matcher name of results") + require.Equal(t, "1.1.1.1", resultEvent.ExtractedResults[0], "could not get correct extracted results") + require.Equal(t, "one.one.one.one", resultEvent.Matched, "could not get matched value") } diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 9c72d9d8..28a3309b 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -4,16 +4,19 @@ import ( "net/url" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" ) var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // Parse the URL and return domain if URL. var domain string if isURL(input) { @@ -23,54 +26,49 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused } // Compile each request for the template based on the URL - compiledRequest, err := r.Make(domain) + compiledRequest, err := request.Make(domain) if err != nil { - r.options.Output.Request(r.options.TemplateID, domain, "dns", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, domain, "dns", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not build request") } requestString := compiledRequest.String() if varErr := expressions.ContainsUnresolvedVariables(requestString); varErr != nil { - gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", r.options.TemplateID, domain, varErr) + gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", request.options.TemplateID, domain, varErr) return nil } - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Str("domain", domain).Msgf("[%s] Dumped DNS request for %s", r.options.TemplateID, domain) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Str("domain", domain).Msgf("[%s] Dumped DNS request for %s", request.options.TemplateID, domain) gologger.Print().Msgf("%s", requestString) } // Send the request to the target servers - resp, err := r.dnsClient.Do(compiledRequest) + resp, err := request.dnsClient.Do(compiledRequest) if err != nil { - r.options.Output.Request(r.options.TemplateID, domain, "dns", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, domain, "dns", err) + request.options.Progress.IncrementFailedRequestsBy(1) } if resp == nil { return errors.Wrap(err, "could not send dns request") } - r.options.Progress.IncrementRequests() + request.options.Progress.IncrementRequests() - r.options.Output.Request(r.options.TemplateID, domain, "dns", err) - gologger.Verbose().Msgf("[%s] Sent DNS request to %s", r.options.TemplateID, domain) + request.options.Output.Request(request.options.TemplateID, domain, "dns", err) + gologger.Verbose().Msgf("[%s] Sent DNS request to %s", request.options.TemplateID, domain) - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain) - gologger.Print().Msgf("%s", resp.String()) - } - outputEvent := r.responseToDSLMap(compiledRequest, resp, input, input) + outputEvent := request.responseToDSLMap(compiledRequest, resp, input, input) for k, v := range previous { outputEvent[k] = v } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = r.MakeResultEvent(event) - } + event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) + + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, resp.String(), request.options.Options.NoColor)) } + callback(event) return nil } diff --git a/v2/pkg/protocols/file/file.go b/v2/pkg/protocols/file/file.go index 2d1e6439..e6196502 100644 --- a/v2/pkg/protocols/file/file.go +++ b/v2/pkg/protocols/file/file.go @@ -56,54 +56,54 @@ type Request struct { var defaultDenylist = []string{".3g2", ".3gp", ".7z", ".apk", ".arj", ".avi", ".axd", ".bmp", ".css", ".csv", ".deb", ".dll", ".doc", ".drv", ".eot", ".exe", ".flv", ".gif", ".gifv", ".gz", ".h264", ".ico", ".iso", ".jar", ".jpeg", ".jpg", ".lock", ".m4a", ".m4v", ".map", ".mkv", ".mov", ".mp3", ".mp4", ".mpeg", ".mpg", ".msi", ".ogg", ".ogm", ".ogv", ".otf", ".pdf", ".pkg", ".png", ".ppt", ".psd", ".rar", ".rm", ".rpm", ".svg", ".swf", ".sys", ".tar.gz", ".tar", ".tif", ".tiff", ".ttf", ".vob", ".wav", ".webm", ".wmv", ".woff", ".woff2", ".xcf", ".xls", ".xlsx", ".zip"} // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { - return r.ID +func (request *Request) GetID() string { + return request.ID } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators +func (request *Request) Compile(options *protocols.ExecuterOptions) error { + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } // By default, use 5 MB as max size to read. - if r.MaxSize == 0 { - r.MaxSize = 5 * 1024 * 1024 + if request.MaxSize == 0 { + request.MaxSize = 5 * 1024 * 1024 } - r.options = options + request.options = options - r.extensions = make(map[string]struct{}) - r.extensionDenylist = make(map[string]struct{}) + request.extensions = make(map[string]struct{}) + request.extensionDenylist = make(map[string]struct{}) - for _, extension := range r.Extensions { + for _, extension := range request.Extensions { if extension == "all" { - r.allExtensions = true + request.allExtensions = true } else { if !strings.HasPrefix(extension, ".") { extension = "." + extension } - r.extensions[extension] = struct{}{} + request.extensions[extension] = struct{}{} } } for _, extension := range defaultDenylist { if !strings.HasPrefix(extension, ".") { extension = "." + extension } - r.extensionDenylist[extension] = struct{}{} + request.extensionDenylist[extension] = struct{}{} } - for _, extension := range r.ExtensionDenylist { + for _, extension := range request.ExtensionDenylist { if !strings.HasPrefix(extension, ".") { extension = "." + extension } - r.extensionDenylist[extension] = struct{}{} + request.extensionDenylist[extension] = struct{}{} } return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { +func (request *Request) Requests() int { return 1 } diff --git a/v2/pkg/protocols/file/find.go b/v2/pkg/protocols/file/find.go index ec1cc258..f4deaa10 100644 --- a/v2/pkg/protocols/file/find.go +++ b/v2/pkg/protocols/file/find.go @@ -7,50 +7,51 @@ import ( "github.com/karrick/godirwalk" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" ) // getInputPaths parses the specified input paths and returns a compiled // list of finished absolute paths to the files evaluating any allowlist, denylist, // glob, file or folders, etc. -func (r *Request) getInputPaths(target string, callback func(string)) error { +func (request *Request) getInputPaths(target string, callback func(string)) error { processed := make(map[string]struct{}) // Template input includes a wildcard - if strings.Contains(target, "*") && !r.NoRecursive { - if err := r.findGlobPathMatches(target, processed, callback); err != nil { + if strings.Contains(target, "*") && !request.NoRecursive { + if err := request.findGlobPathMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find glob matches") } return nil } // Template input is either a file or a directory - file, err := r.findFileMatches(target, processed, callback) + file, err := request.findFileMatches(target, processed, callback) if err != nil { return errors.Wrap(err, "could not find file") } if file { return nil } - if r.NoRecursive { + if request.NoRecursive { return nil // we don't process dirs in no-recursive mode } // Recursively walk down the Templates directory and run all // the template file checks - if err := r.findDirectoryMatches(target, processed, callback); err != nil { + if err := request.findDirectoryMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find directory matches") } return nil } // findGlobPathMatches returns the matched files from a glob path -func (r *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error { +func (request *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error { matches, err := filepath.Glob(absPath) if err != nil { return errors.Errorf("wildcard found, but unable to glob: %s\n", err) } for _, match := range matches { - if !r.validatePath(match) { + if !request.validatePath(match) { continue } if _, ok := processed[match]; !ok { @@ -63,7 +64,7 @@ func (r *Request) findGlobPathMatches(absPath string, processed map[string]struc // findFileMatches finds if a path is an absolute file. If the path // is a file, it returns true otherwise false with no errors. -func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) { +func (request *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) { info, err := os.Stat(absPath) if err != nil { return false, err @@ -72,7 +73,7 @@ func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, return false, nil } if _, ok := processed[absPath]; !ok { - if !r.validatePath(absPath) { + if !request.validatePath(absPath) { return false, nil } processed[absPath] = struct{}{} @@ -82,7 +83,7 @@ func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, } // findDirectoryMatches finds matches for templates from a directory -func (r *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error { +func (request *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error { err := godirwalk.Walk(absPath, &godirwalk.Options{ Unsorted: true, ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction { @@ -92,7 +93,7 @@ func (r *Request) findDirectoryMatches(absPath string, processed map[string]stru if d.IsDir() { return nil } - if !r.validatePath(path) { + if !request.validatePath(path) { return nil } if _, ok := processed[path]; !ok { @@ -106,17 +107,17 @@ func (r *Request) findDirectoryMatches(absPath string, processed map[string]stru } // validatePath validates a file path for blacklist and whitelist options -func (r *Request) validatePath(item string) bool { +func (request *Request) validatePath(item string) bool { extension := filepath.Ext(item) - if len(r.extensions) > 0 { - if _, ok := r.extensions[extension]; ok { + if len(request.extensions) > 0 { + if _, ok := request.extensions[extension]; ok { return true - } else if !r.allExtensions { + } else if !request.allExtensions { return false } } - if _, ok := r.extensionDenylist[extension]; ok { + if _, ok := request.extensionDenylist[extension]; ok { gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, extension) return false } diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index 743dd7d1..c0d95c5b 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -6,14 +6,16 @@ import ( "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "all", "data", "": @@ -22,27 +24,27 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(itemStr))) + return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr, nil)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { partString := extractor.Part switch partString { case "body", "all", "data", "": @@ -64,49 +66,27 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext return nil } -// responseToDSLMap converts a DNS response to a map for use in DSL matching -func (r *Request) responseToDSLMap(raw, host, matched string) output.InternalEvent { - data := make(output.InternalEvent, 5) - - // Some data regarding the request metadata - data["path"] = host - data["matched"] = matched - data["raw"] = raw - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath - return data +// responseToDSLMap converts a file response to a map for use in DSL matching +func (request *Request) responseToDSLMap(raw, inputFilePath, matchedFileName string) output.InternalEvent { + return output.InternalEvent{ + "path": inputFilePath, + "matched": matchedFileName, + "raw": raw, + "template-id": request.options.TemplateID, + "template-info": request.options.TemplateInfo, + "template-path": request.options.TemplatePath, + } } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + results := protocols.MakeDefaultResultEvent(request, wrapped) - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := r.makeResultEventItem(wrapped) - results = append(results, data) - } raw, ok := wrapped.InternalEvent["raw"] if !ok { return results } + rawStr, ok := raw.(string) if !ok { return results @@ -133,7 +113,11 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu return results } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), @@ -141,7 +125,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out Type: "file", Path: types.ToString(wrapped.InternalEvent["path"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), - Host: types.ToString(wrapped.InternalEvent["matched"]), + Host: types.ToString(wrapped.InternalEvent["host"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Response: types.ToString(wrapped.InternalEvent["raw"]), Timestamp: time.Now(), diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index 7c5be06d..b03d641b 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -72,8 +72,9 @@ func TestFileOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -86,8 +87,9 @@ func TestFileOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -99,8 +101,9 @@ func TestFileOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } @@ -156,7 +159,62 @@ func TestFileOperatorExtract(t *testing.T) { }) } -func TestFileMakeResult(t *testing.T) { +func TestFileMakeResultWithOrMatcher(t *testing.T) { + expectedValue := []string{"1.1.1.1"} + namedMatcherName := "test" + + finalEvent := testFileMakeResultOperators(t, "or") + require.Equal(t, namedMatcherName, finalEvent.Results[0].MatcherName) + require.Equal(t, expectedValue, finalEvent.OperatorsResult.Matches[namedMatcherName], "could not get matched value") +} + +func TestFileMakeResultWithAndMatcher(t *testing.T) { + finalEvent := testFileMakeResultOperators(t, "and") + require.Equal(t, "", finalEvent.Results[0].MatcherName) + require.Empty(t, finalEvent.OperatorsResult.Matches) +} + +func testFileMakeResultOperators(t *testing.T, matcherCondition string) *output.InternalWrappedEvent { + expectedValue := []string{"1.1.1.1"} + namedMatcherName := "test" + matcher := []*matchers.Matcher{ + { + Part: "raw", + Type: "word", + Words: expectedValue, + }, + { + Name: namedMatcherName, + Part: "raw", + Type: "word", + Words: expectedValue, + }, + } + + expectedValues := map[string][]string{ + "word-1": expectedValue, + namedMatcherName: expectedValue, + } + + finalEvent := testFileMakeResult(t, matcher, matcherCondition, true) + for matcherName, matchedValues := range expectedValues { + var matchesOne = false + for i := 0; i <= len(expectedValue); i++ { + resultEvent := finalEvent.Results[i] + if matcherName == resultEvent.MatcherName { + matchesOne = true + } + } + require.True(t, matchesOne) + require.Equal(t, matchedValues, finalEvent.OperatorsResult.Matches[matcherName], "could not get matched value") + } + + finalEvent = testFileMakeResult(t, matcher, matcherCondition, false) + require.Equal(t, 1, len(finalEvent.Results)) + return finalEvent +} + +func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondition string, isDebug bool) *output.InternalWrappedEvent { options := testutils.DefaultOptions testutils.Init(options) @@ -168,12 +226,8 @@ func TestFileMakeResult(t *testing.T) { Extensions: []string{"*", ".lock"}, ExtensionDenylist: []string{".go"}, Operators: operators.Operators{ - Matchers: []*matchers.Matcher{{ - Name: "test", - Part: "raw", - Type: "word", - Words: []string{"1.1.1.1"}, - }}, + MatchersCondition: matcherCondition, + Matchers: matchers, Extractors: []*extractors.Extractor{{ Part: "raw", Type: "regex", @@ -188,20 +242,24 @@ func TestFileMakeResult(t *testing.T) { err := request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") - resp := "test-data\r\n1.1.1.1\r\n" - event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") + matchedFileName := "test.txt" + fileContent := "test-data\r\n1.1.1.1\r\n" + + event := request.responseToDSLMap(fileContent, "/tmp", matchedFileName) require.Len(t, event, 6, "could not get correct number of items in dsl map") - require.Equal(t, resp, event["raw"], "could not get correct resp") + require.Equal(t, fileContent, event["raw"], "could not get correct resp") finalEvent := &output.InternalWrappedEvent{InternalEvent: event} if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract) + result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, isDebug) if ok && result != nil { finalEvent.OperatorsResult = result finalEvent.Results = request.MakeResultEvent(finalEvent) } } - require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results") - require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results") - require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results") + resultEvent := finalEvent.Results[0] + require.Equal(t, "1.1.1.1", resultEvent.ExtractedResults[0], "could not get correct extracted results") + require.Equal(t, matchedFileName, resultEvent.Matched, "could not get matched value") + + return finalEvent } diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index d272e256..61bb8718 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -5,75 +5,74 @@ import ( "os" "github.com/pkg/errors" + "github.com/remeh/sizedwaitgroup" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" - "github.com/remeh/sizedwaitgroup" ) var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - wg := sizedwaitgroup.New(r.options.Options.BulkSize) +func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + wg := sizedwaitgroup.New(request.options.Options.BulkSize) - err := r.getInputPaths(input, func(data string) { + err := request.getInputPaths(input, func(data string) { wg.Add() - go func(data string) { + go func(filePath string) { defer wg.Done() - file, err := os.Open(data) + file, err := os.Open(filePath) if err != nil { - gologger.Error().Msgf("Could not open file path %s: %s\n", data, err) + gologger.Error().Msgf("Could not open file path %s: %s\n", filePath, err) return } defer file.Close() stat, err := file.Stat() if err != nil { - gologger.Error().Msgf("Could not stat file path %s: %s\n", data, err) + gologger.Error().Msgf("Could not stat file path %s: %s\n", filePath, err) return } - if stat.Size() >= int64(r.MaxSize) { - gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", data) + if stat.Size() >= int64(request.MaxSize) { + gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", filePath) return } buffer, err := ioutil.ReadAll(file) if err != nil { - gologger.Error().Msgf("Could not read file path %s: %s\n", data, err) + gologger.Error().Msgf("Could not read file path %s: %s\n", filePath, err) return } dataStr := tostring.UnsafeToString(buffer) - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped file request for %s", r.options.TemplateID, data) - gologger.Print().Msgf("%s", dataStr) - } - gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, data) - outputEvent := r.responseToDSLMap(dataStr, input, data) + + gologger.Verbose().Msgf("[%s] Sent FILE request to %s", request.options.TemplateID, filePath) + outputEvent := request.responseToDSLMap(dataStr, input, filePath) for k, v := range previous { outputEvent[k] = v } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = r.MakeResultEvent(event) - } + event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) + + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, dataStr, request.options.Options.NoColor)) } + callback(event) }(data) }) wg.Wait() if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "file", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "file", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } - r.options.Progress.IncrementRequests() + request.options.Progress.IncrementRequests() return nil } diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index 8af250b2..43f62f43 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -32,24 +32,24 @@ type Step struct { } // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { - return r.ID +func (request *Request) GetID() string { + return request.ID } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators +func (request *Request) Compile(options *protocols.ExecuterOptions) error { + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } - r.options = options + request.options = options return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { +func (request *Request) Requests() int { return 1 } diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index c384ee26..81a81a6a 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -4,14 +4,16 @@ import ( "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "resp", "": @@ -20,27 +22,27 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(itemStr))) + return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr, nil)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { partString := extractor.Part switch partString { case "body", "resp", "": @@ -62,50 +64,29 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext return nil } -// responseToDSLMap converts a DNS response to a map for use in DSL matching -func (r *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent { - data := make(output.InternalEvent, 5) - - // Some data regarding the request metadata - data["host"] = host - data["matched"] = matched - data["req"] = req - data["data"] = resp - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath - return data +// responseToDSLMap converts a headless response to a map for use in DSL matching +func (request *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent { + return output.InternalEvent{ + "host": host, + "matched": matched, + "req": req, + "data": resp, + "template-id": request.options.TemplateID, + "template-info": request.options.TemplateInfo, + "template-path": request.options.TemplatePath, + } } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := r.makeResultEventItem(wrapped) - results = append(results, data) - } - return results +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 54345d46..55394ed7 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -6,75 +6,72 @@ import ( "time" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" ) var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { - instance, err := r.options.Browser.NewInstance() +func (request *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { + instance, err := request.options.Browser.NewInstance() if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "headless", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } defer instance.Close() parsed, err := url.Parse(input) if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "headless", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } - out, page, err := instance.Run(parsed, r.Steps, time.Duration(r.options.Options.PageTimeout)*time.Second) + out, page, err := instance.Run(parsed, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second) if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "headless", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } defer page.Close() - r.options.Output.Request(r.options.TemplateID, input, "headless", nil) - r.options.Progress.IncrementRequests() + request.options.Output.Request(request.options.TemplateID, input, "headless", nil) + request.options.Progress.IncrementRequests() gologger.Verbose().Msgf("Sent Headless request to %s", input) reqBuilder := &strings.Builder{} - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped Headless request for %s", r.options.TemplateID, input) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, input) - for _, act := range r.Steps { + for _, act := range request.Steps { reqBuilder.WriteString(act.String()) reqBuilder.WriteString("\n") } gologger.Print().Msgf("%s", reqBuilder.String()) } - var respBody string + var responseBody string html, err := page.Page().Element("html") if err == nil { - respBody, _ = html.HTML() + responseBody, _ = html.HTML() } - outputEvent := r.responseToDSLMap(respBody, reqBuilder.String(), input, input) + outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), input, input) for k, v := range out { outputEvent[k] = v } - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Headless response for %s", r.options.TemplateID, input) - gologger.Print().Msgf("%s", respBody) + event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) + + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor)) } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = r.MakeResultEvent(event) - } - } callback(event) return nil } diff --git a/v2/pkg/protocols/http/cluster.go b/v2/pkg/protocols/http/cluster.go index bdfa0735..8c1674be 100644 --- a/v2/pkg/protocols/http/cluster.go +++ b/v2/pkg/protocols/http/cluster.go @@ -9,20 +9,20 @@ import ( // This used by the clustering engine to decide whether two requests // are similar enough to be considered one and can be checked by // just adding the matcher/extractors for the request and the correct IDs. -func (r *Request) CanCluster(other *Request) bool { - if len(r.Payloads) > 0 || len(r.Raw) > 0 || len(r.Body) > 0 || r.Unsafe || r.ReqCondition || r.Name != "" { +func (request *Request) CanCluster(other *Request) bool { + if len(request.Payloads) > 0 || len(request.Raw) > 0 || len(request.Body) > 0 || request.Unsafe || request.ReqCondition || request.Name != "" { return false } - if r.Method != other.Method || - r.MaxRedirects != other.MaxRedirects || - r.CookieReuse != other.CookieReuse || - r.Redirects != other.Redirects { + if request.Method != other.Method || + request.MaxRedirects != other.MaxRedirects || + request.CookieReuse != other.CookieReuse || + request.Redirects != other.Redirects { return false } - if !compare.StringSlice(r.Path, other.Path) { + if !compare.StringSlice(request.Path, other.Path) { return false } - if !compare.StringMap(r.Headers, other.Headers) { + if !compare.StringMap(request.Headers, other.Headers) { return false } return true diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 69738e36..67dc173b 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -176,21 +176,21 @@ type Request struct { } // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { - return r.ID +func (request *Request) GetID() string { + return request.ID } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { +func (request *Request) Compile(options *protocols.ExecuterOptions) error { connectionConfiguration := &httpclientpool.Configuration{ - Threads: r.Threads, - MaxRedirects: r.MaxRedirects, - FollowRedirects: r.Redirects, - CookieReuse: r.CookieReuse, + Threads: request.Threads, + MaxRedirects: request.MaxRedirects, + FollowRedirects: request.Redirects, + CookieReuse: request.CookieReuse, } // if the headers contain "Connection" we need to disable the automatic keep alive of the standard library - if _, hasConnectionHeader := r.Headers["Connection"]; hasConnectionHeader { + if _, hasConnectionHeader := request.Headers["Connection"]; hasConnectionHeader { connectionConfiguration.Connection = &httpclientpool.ConnectionConfiguration{DisableKeepAlive: false} } @@ -198,49 +198,49 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if err != nil { return errors.Wrap(err, "could not get dns client") } - r.customHeaders = make(map[string]string) - r.httpClient = client - r.options = options - for _, option := range r.options.Options.CustomHeaders { + request.customHeaders = make(map[string]string) + request.httpClient = client + request.options = options + for _, option := range request.options.Options.CustomHeaders { parts := strings.SplitN(option, ":", 2) if len(parts) != 2 { continue } - r.customHeaders[parts[0]] = strings.TrimSpace(parts[1]) + request.customHeaders[parts[0]] = strings.TrimSpace(parts[1]) } - if r.Body != "" && !strings.Contains(r.Body, "\r\n") { - r.Body = strings.ReplaceAll(r.Body, "\n", "\r\n") + if request.Body != "" && !strings.Contains(request.Body, "\r\n") { + request.Body = strings.ReplaceAll(request.Body, "\n", "\r\n") } - if len(r.Raw) > 0 { - for i, raw := range r.Raw { + if len(request.Raw) > 0 { + for i, raw := range request.Raw { if !strings.Contains(raw, "\r\n") { - r.Raw[i] = strings.ReplaceAll(raw, "\n", "\r\n") + request.Raw[i] = strings.ReplaceAll(raw, "\n", "\r\n") } } - r.rawhttpClient = httpclientpool.GetRawHTTP(options.Options) + request.rawhttpClient = httpclientpool.GetRawHTTP(options.Options) } - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if compileErr := compiled.Compile(); compileErr != nil { return errors.Wrap(compileErr, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } // Resolve payload paths from vars if they exists - for name, payload := range r.options.Options.VarsPayload() { + for name, payload := range request.options.Options.VarsPayload() { payloadStr, ok := payload.(string) // check if inputs contains the payload var hasPayloadName bool // search for markers in all request parts var inputs []string - inputs = append(inputs, r.Method, r.Body) - inputs = append(inputs, r.Raw...) - for k, v := range r.customHeaders { + inputs = append(inputs, request.Method, request.Body) + inputs = append(inputs, request.Raw...) + for k, v := range request.customHeaders { inputs = append(inputs, fmt.Sprintf("%s: %s", k, v)) } - for k, v := range r.Headers { + for k, v := range request.Headers { inputs = append(inputs, fmt.Sprintf("%s: %s", k, v)) } @@ -251,57 +251,57 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } } if ok && hasPayloadName && fileutil.FileExists(payloadStr) { - if r.Payloads == nil { - r.Payloads = make(map[string]interface{}) + if request.Payloads == nil { + request.Payloads = make(map[string]interface{}) } - r.Payloads[name] = payloadStr + request.Payloads[name] = payloadStr } } - if len(r.Payloads) > 0 { - attackType := r.AttackType + if len(request.Payloads) > 0 { + attackType := request.AttackType if attackType == "" { attackType = "batteringram" } var ok bool - r.attackType, ok = generators.StringToType[attackType] + request.attackType, ok = generators.StringToType[attackType] if !ok { return fmt.Errorf("invalid attack type provided: %s", attackType) } // Resolve payload paths if they are files. - for name, payload := range r.Payloads { + for name, payload := range request.Payloads { payloadStr, ok := payload.(string) if ok { final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) if resolveErr != nil { return errors.Wrap(resolveErr, "could not read payload file") } - r.Payloads[name] = final + request.Payloads[name] = final } } - r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath) + request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath) if err != nil { return errors.Wrap(err, "could not parse payloads") } } - r.options = options - r.totalRequests = r.Requests() + request.options = options + request.totalRequests = request.Requests() return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { - if r.generator != nil { - payloadRequests := r.generator.NewIterator().Total() * len(r.Raw) +func (request *Request) Requests() int { + if request.generator != nil { + payloadRequests := request.generator.NewIterator().Total() * len(request.Raw) return payloadRequests } - if len(r.Raw) > 0 { - requests := len(r.Raw) - if requests == 1 && r.RaceNumberRequests != 0 { - requests *= r.RaceNumberRequests + if len(request.Raw) > 0 { + requests := len(request.Raw) + if requests == 1 && request.RaceNumberRequests != 0 { + requests *= request.RaceNumberRequests } return requests } - return len(r.Path) + return len(request.Path) } diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index a836fba7..c91ee0ec 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -6,46 +6,57 @@ import ( "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { item, ok := getMatchPart(matcher.Part, data) if !ok { - return false + return false, []string{} } switch matcher.GetType() { case matchers.StatusMatcher: - statusCode, ok := data["status_code"] + statusCode, ok := getStatusCode(data) if !ok { - return false + return false, []string{} } - status, ok := statusCode.(int) - if !ok { - return false - } - return matcher.Result(matcher.MatchStatusCode(status)) + return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{responsehighlighter.CreateStatusCodeSnippet(data["response"].(string), statusCode)} case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(item))) + return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(item, r.dynamicValues)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, request.dynamicValues)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} +} + +func getStatusCode(data map[string]interface{}) (int, bool) { + statusCodeValue, ok := data["status_code"] + if !ok { + return 0, false + } + statusCode, ok := statusCodeValue.(int) + if !ok { + return 0, false + } + return statusCode, true } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { item, ok := getMatchPart(extractor.Part, data) if !ok { return nil @@ -86,8 +97,8 @@ func getMatchPart(part string, data output.InternalEvent) (string, bool) { } // responseToDSLMap converts an HTTP response to a map for use in DSL matching -func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { - data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies())) +func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) output.InternalEvent { + data := make(output.InternalEvent, 12+len(extra)+len(resp.Header)+len(resp.Cookies())) for k, v := range extra { data[k] = v } @@ -107,42 +118,22 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r data["content_length"] = resp.ContentLength data["all_headers"] = headers data["duration"] = duration.Seconds() - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath + data["template-id"] = request.options.TemplateID + data["template-info"] = request.options.TemplateInfo + data["template-path"] = request.options.TemplatePath return data } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 && !wrapped.OperatorsResult.Matched { - return nil - } - - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := r.makeResultEventItem(wrapped) - results = append(results, data) - } - return results +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index 6c24f236..c9d3b795 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -84,8 +84,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -98,8 +99,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -111,8 +113,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } @@ -259,7 +262,7 @@ func TestHTTPMakeResult(t *testing.T) { event["ip"] = "192.169.1.1" finalEvent := &output.InternalWrappedEvent{InternalEvent: event} if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract) + result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false) if ok && result != nil { finalEvent.OperatorsResult = result finalEvent.Results = request.MakeResultEvent(finalEvent) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 0111de42..979e0eae 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -21,6 +21,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" @@ -31,52 +33,52 @@ import ( const defaultMaxWorkers = 150 // executeRaceRequest executes race condition request for a URL -func (r *Request) executeRaceRequest(reqURL string, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - var requests []*generatedRequest +func (request *Request) executeRaceRequest(reqURL string, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + var generatedRequests []*generatedRequest // Requests within race condition should be dumped once and the output prefilled to allow DSL language to work // This will introduce a delay and will populate in hacky way the field "request" of outputEvent - generator := r.newGenerator() + generator := request.newGenerator() requestForDump, err := generator.Make(reqURL, nil, "") if err != nil { return err } - r.setCustomHeaders(requestForDump) + request.setCustomHeaders(requestForDump) dumpedRequest, err := dump(requestForDump, reqURL) if err != nil { return err } - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL) gologger.Print().Msgf("%s", string(dumpedRequest)) } previous["request"] = string(dumpedRequest) // Pre-Generate requests - for i := 0; i < r.RaceNumberRequests; i++ { - generator := r.newGenerator() - request, err := generator.Make(reqURL, nil, "") + for i := 0; i < request.RaceNumberRequests; i++ { + generator := request.newGenerator() + generatedRequest, err := generator.Make(reqURL, nil, "") if err != nil { return err } - requests = append(requests, request) + generatedRequests = append(generatedRequests, generatedRequest) } wg := sync.WaitGroup{} var requestErr error mutex := &sync.Mutex{} - for i := 0; i < r.RaceNumberRequests; i++ { + for i := 0; i < request.RaceNumberRequests; i++ { wg.Add(1) go func(httpRequest *generatedRequest) { defer wg.Done() - err := r.executeRequest(reqURL, httpRequest, previous, false, callback, 0) + err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) } mutex.Unlock() - }(requests[i]) - r.options.Progress.IncrementRequests() + }(generatedRequests[i]) + request.options.Progress.IncrementRequests() } wg.Wait() @@ -84,47 +86,47 @@ func (r *Request) executeRaceRequest(reqURL string, previous output.InternalEven } // executeRaceRequest executes parallel requests for a template -func (r *Request) executeParallelHTTP(reqURL string, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error { - generator := r.newGenerator() +func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error { + generator := request.newGenerator() // Workers that keeps enqueuing new requests - maxWorkers := r.Threads + maxWorkers := request.Threads swg := sizedwaitgroup.New(maxWorkers) var requestErr error mutex := &sync.Mutex{} for { - request, err := generator.Make(reqURL, dynamicValues, "") + generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "") if err == io.EOF { break } if err != nil { - r.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err } swg.Add() go func(httpRequest *generatedRequest) { defer swg.Done() - r.options.RateLimiter.Take() + request.options.RateLimiter.Take() previous := make(map[string]interface{}) - err := r.executeRequest(reqURL, httpRequest, previous, false, callback, 0) + err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) } mutex.Unlock() - }(request) - r.options.Progress.IncrementRequests() + }(generatedHttpRequest) + request.options.Progress.IncrementRequests() } swg.Wait() return requestErr } // executeTurboHTTP executes turbo http request for a URL -func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - generator := r.newGenerator() +func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + generator := request.newGenerator() // need to extract the target from the url URL, err := url.Parse(reqURL) @@ -135,13 +137,13 @@ func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output pipeOptions := rawhttp.DefaultPipelineOptions pipeOptions.Host = URL.Host pipeOptions.MaxConnections = 1 - if r.PipelineConcurrentConnections > 0 { - pipeOptions.MaxConnections = r.PipelineConcurrentConnections + if request.PipelineConcurrentConnections > 0 { + pipeOptions.MaxConnections = request.PipelineConcurrentConnections } - if r.PipelineRequestsPerConnection > 0 { - pipeOptions.MaxPendingRequests = r.PipelineRequestsPerConnection + if request.PipelineRequestsPerConnection > 0 { + pipeOptions.MaxPendingRequests = request.PipelineRequestsPerConnection } - pipeclient := rawhttp.NewPipelineClient(pipeOptions) + pipeClient := rawhttp.NewPipelineClient(pipeOptions) // defaultMaxWorkers should be a sufficient value to keep queues always full maxWorkers := defaultMaxWorkers @@ -154,90 +156,90 @@ func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output var requestErr error mutex := &sync.Mutex{} for { - request, err := generator.Make(reqURL, dynamicValues, "") + generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "") if err == io.EOF { break } if err != nil { - r.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err } - request.pipelinedClient = pipeclient + generatedHttpRequest.pipelinedClient = pipeClient swg.Add() go func(httpRequest *generatedRequest) { defer swg.Done() - err := r.executeRequest(reqURL, httpRequest, previous, false, callback, 0) + err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) } mutex.Unlock() - }(request) - r.options.Progress.IncrementRequests() + }(generatedHttpRequest) + request.options.Progress.IncrementRequests() } swg.Wait() return requestErr } // ExecuteWithResults executes the final request on a URL -func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // verify if pipeline was requested - if r.Pipeline { - return r.executeTurboHTTP(reqURL, dynamicValues, previous, callback) + if request.Pipeline { + return request.executeTurboHTTP(reqURL, dynamicValues, previous, callback) } // verify if a basic race condition was requested - if r.Race && r.RaceNumberRequests > 0 { - return r.executeRaceRequest(reqURL, previous, callback) + if request.Race && request.RaceNumberRequests > 0 { + return request.executeRaceRequest(reqURL, previous, callback) } // verify if parallel elaboration was requested - if r.Threads > 0 { - return r.executeParallelHTTP(reqURL, dynamicValues, callback) + if request.Threads > 0 { + return request.executeParallelHTTP(reqURL, dynamicValues, callback) } - generator := r.newGenerator() + generator := request.newGenerator() requestCount := 1 var requestErr error for { - hasInteractMarkers := interactsh.HasMatchers(r.CompiledOperators) + hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators) var interactURL string - if r.options.Interactsh != nil && hasInteractMarkers { - interactURL = r.options.Interactsh.URL() + if request.options.Interactsh != nil && hasInteractMarkers { + interactURL = request.options.Interactsh.URL() } - request, err := generator.Make(reqURL, dynamicValues, interactURL) + generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, interactURL) if err == io.EOF { break } if err != nil { - r.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err } - r.dynamicValues = request.dynamicValues + request.dynamicValues = generatedHttpRequest.dynamicValues // Check if hosts just keep erroring - if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.Check(reqURL) { + if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) { break } var gotOutput bool - r.options.RateLimiter.Take() - err = r.executeRequest(reqURL, request, previous, hasInteractMarkers, func(event *output.InternalWrappedEvent) { + request.options.RateLimiter.Take() + err = request.executeRequest(reqURL, generatedHttpRequest, previous, hasInteractMarkers, func(event *output.InternalWrappedEvent) { // Add the extracts to the dynamic values if any. if event.OperatorsResult != nil { gotOutput = true dynamicValues = generators.MergeMaps(dynamicValues, event.OperatorsResult.DynamicValues) } - if hasInteractMarkers && r.options.Interactsh != nil { - r.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ - MakeResultFunc: r.MakeResultEvent, + if hasInteractMarkers && request.options.Interactsh != nil { + request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ + MakeResultFunc: request.MakeResultEvent, Event: event, - Operators: r.CompiledOperators, - MatchFunc: r.Match, - ExtractFunc: r.Extract, + Operators: request.CompiledOperators, + MatchFunc: request.Match, + ExtractFunc: request.Extract, }) } else { callback(event) @@ -248,16 +250,16 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp break } if err != nil { - if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.CheckError(err) { - r.options.HostErrorsCache.MarkFailed(reqURL) + if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.CheckError(err) { + request.options.HostErrorsCache.MarkFailed(reqURL) } requestErr = err } requestCount++ - r.options.Progress.IncrementRequests() + request.options.Progress.IncrementRequests() // If this was a match and we want to stop at first match, skip all further requests. - if (request.original.options.Options.StopAtFirstMatch || r.StopAtFirstMatch) && gotOutput { + if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || request.StopAtFirstMatch) && gotOutput { break } } @@ -269,33 +271,33 @@ const drainReqSize = int64(8 * 1024) var errStopExecution = errors.New("stop execution due to unresolved variables") // executeRequest executes the actual generated request and returns error if occurred -func (r *Request) executeRequest(reqURL string, request *generatedRequest, previous output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error { - r.setCustomHeaders(request) +func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error { + request.setCustomHeaders(generatedRequest) var ( resp *http.Response - fromcache bool + fromCache bool dumpedRequest []byte err error ) // For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function - if !request.original.Race { + if !generatedRequest.original.Race { var dumpError error - dumpedRequest, dumpError = dump(request, reqURL) + dumpedRequest, dumpError = dump(generatedRequest, reqURL) if dumpError != nil { return dumpError } dumpedRequestString := string(dumpedRequest) // Check if are there any unresolved variables. If yes, skip unless overriden by user. - if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !r.SkipVariablesCheck { - gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", r.options.TemplateID, reqURL, varErr) + if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { + gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr) return errStopExecution } - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL) gologger.Print().Msgf("%s", dumpedRequestString) } } @@ -303,61 +305,61 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ var formedURL string var hostname string timeStart := time.Now() - if request.original.Pipeline { - if request.rawRequest != nil { - formedURL = request.rawRequest.FullURL + if generatedRequest.original.Pipeline { + if generatedRequest.rawRequest != nil { + formedURL = generatedRequest.rawRequest.FullURL if parsed, parseErr := url.Parse(formedURL); parseErr == nil { hostname = parsed.Host } - resp, err = request.pipelinedClient.DoRaw(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data))) - } else if request.request != nil { - resp, err = request.pipelinedClient.Dor(request.request) + resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, reqURL, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data))) + } else if generatedRequest.request != nil { + resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request) } - } else if request.original.Unsafe && request.rawRequest != nil { - formedURL = request.rawRequest.FullURL + } else if generatedRequest.original.Unsafe && generatedRequest.rawRequest != nil { + formedURL = generatedRequest.rawRequest.FullURL if parsed, parseErr := url.Parse(formedURL); parseErr == nil { hostname = parsed.Host } - options := request.original.rawhttpClient.Options - options.FollowRedirects = r.Redirects - options.CustomRawBytes = request.rawRequest.UnsafeRawBytes - resp, err = request.original.rawhttpClient.DoRawWithOptions(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data)), options) + options := generatedRequest.original.rawhttpClient.Options + options.FollowRedirects = request.Redirects + options.CustomRawBytes = generatedRequest.rawRequest.UnsafeRawBytes + resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, reqURL, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), options) } else { - hostname = request.request.URL.Host - formedURL = request.request.URL.String() + hostname = generatedRequest.request.URL.Host + formedURL = generatedRequest.request.URL.String() // if nuclei-project is available check if the request was already sent previously - if r.options.ProjectFile != nil { + if request.options.ProjectFile != nil { // if unavailable fail silently - fromcache = true - resp, err = r.options.ProjectFile.Get(dumpedRequest) + fromCache = true + resp, err = request.options.ProjectFile.Get(dumpedRequest) if err != nil { - fromcache = false + fromCache = false } } if resp == nil { - resp, err = r.httpClient.Do(request.request) + resp, err = request.httpClient.Do(generatedRequest.request) } } if err != nil { // rawhttp doesn't support draining response bodies. - if resp != nil && resp.Body != nil && request.rawRequest == nil { + if resp != nil && resp.Body != nil && generatedRequest.rawRequest == nil { _, _ = io.CopyN(ioutil.Discard, resp.Body, drainReqSize) resp.Body.Close() } - r.options.Output.Request(r.options.TemplateID, formedURL, "http", err) - r.options.Progress.IncrementErrorsBy(1) + request.options.Output.Request(request.options.TemplateID, formedURL, "http", err) + request.options.Progress.IncrementErrorsBy(1) // If we have interactsh markers and request times out, still send // a callback event so in case we receive an interaction, correlation is possible. if hasInteractMarkers { - outputEvent := r.responseToDSLMap(&http.Response{}, reqURL, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, request.meta) + outputEvent := request.responseToDSLMap(&http.Response{}, reqURL, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { + if request.CompiledOperators != nil { event.InternalEvent = outputEvent } callback(event) @@ -371,8 +373,8 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ resp.Body.Close() }() - gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", r.options.TemplateID, formedURL) - r.options.Output.Request(r.options.TemplateID, formedURL, "http", err) + gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", request.options.TemplateID, formedURL) + request.options.Output.Request(request.options.TemplateID, formedURL, "http", err) duration := time.Since(timeStart) @@ -385,8 +387,8 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ // If the status code is HTTP 101, we should not proceed with reading body. if resp.StatusCode != http.StatusSwitchingProtocols { var bodyReader io.Reader - if r.MaxSize != 0 { - bodyReader = io.LimitReader(resp.Body, int64(r.MaxSize)) + if request.MaxSize != 0 { + bodyReader = io.LimitReader(resp.Body, int64(request.MaxSize)) } else { bodyReader = resp.Body } @@ -394,7 +396,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ if err != nil { // Ignore body read due to server misconfiguration errors if stringsutil.ContainsAny(err.Error(), "gzip: invalid header") { - gologger.Warning().Msgf("[%s] Server sent an invalid gzip header and it was not possible to read the uncompressed body for %s: %s", r.options.TemplateID, formedURL, err.Error()) + gologger.Warning().Msgf("[%s] Server sent an invalid gzip header and it was not possible to read the uncompressed body for %s: %s", request.options.TemplateID, formedURL, err.Error()) } else if !stringsutil.ContainsAny(err.Error(), "unexpected EOF", "user canceled") { // ignore EOF and random error return errors.Wrap(err, "could not read http body") } @@ -438,66 +440,59 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ } } - // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, formedURL) - gologger.Print().Msgf("%s", string(redirectedResponse)) - } - // if nuclei-project is enabled store the response if not previously done - if r.options.ProjectFile != nil && !fromcache { - if err := r.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil { + if request.options.ProjectFile != nil && !fromCache { + if err := request.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil { return errors.Wrap(err, "could not store in project file") } } matchedURL := reqURL - if request.rawRequest != nil && request.rawRequest.FullURL != "" { - matchedURL = request.rawRequest.FullURL + if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" { + matchedURL = generatedRequest.rawRequest.FullURL } - if request.request != nil { - matchedURL = request.request.URL.String() + if generatedRequest.request != nil { + matchedURL = generatedRequest.request.URL.String() } finalEvent := make(output.InternalEvent) - outputEvent := r.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, request.meta) + outputEvent := request.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, generatedRequest.meta) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse) - for k, v := range previous { + for k, v := range previousEvent { finalEvent[k] = v } for k, v := range outputEvent { finalEvent[k] = v } // Add to history the current request number metadata if asked by the user. - if r.ReqCondition { + if request.ReqCondition { for k, v := range outputEvent { key := fmt.Sprintf("%s_%d", k, requestCount) - previous[key] = v + previousEvent[key] = v finalEvent[key] = v } } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - var ok bool - event.OperatorsResult, ok = r.CompiledOperators.Execute(finalEvent, r.Match, r.Extract) - if ok && event.OperatorsResult != nil { - event.OperatorsResult.PayloadValues = request.meta - event.Results = r.MakeResultEvent(event) - } - event.InternalEvent = outputEvent + event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta + }) + + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, string(redirectedResponse), request.options.Options.NoColor)) } + callback(event) return nil } // setCustomHeaders sets the custom headers for generated request -func (r *Request) setCustomHeaders(req *generatedRequest) { - for k, v := range r.customHeaders { +func (request *Request) setCustomHeaders(req *generatedRequest) { + for k, v := range request.customHeaders { if req.rawRequest != nil { req.rawRequest.Headers[k] = v } else { diff --git a/v2/pkg/protocols/http/request_generator.go b/v2/pkg/protocols/http/request_generator.go index 23622592..90c5d0fb 100644 --- a/v2/pkg/protocols/http/request_generator.go +++ b/v2/pkg/protocols/http/request_generator.go @@ -19,11 +19,11 @@ type requestGenerator struct { } // newGenerator creates a new request generator instance -func (r *Request) newGenerator() *requestGenerator { - generator := &requestGenerator{request: r, options: r.options} +func (request *Request) newGenerator() *requestGenerator { + generator := &requestGenerator{request: request, options: request.options} - if len(r.Payloads) > 0 { - generator.payloadIterator = r.generator.NewIterator() + if len(request.Payloads) > 0 { + generator.payloadIterator = request.generator.NewIterator() } return generator } @@ -63,7 +63,6 @@ func (r *requestGenerator) nextValue() (value string, payloads map[string]interf } } - if len(r.request.Raw) > 0 && r.currentIndex < len(r.request.Raw) { if r.payloadIterator != nil { payload, ok := r.payloadIterator.Value() diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index b7de7b92..e54a6684 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -116,17 +116,17 @@ type Input struct { } // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { - return r.ID +func (request *Request) GetID() string { + return request.ID } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { +func (request *Request) Compile(options *protocols.ExecuterOptions) error { var shouldUseTLS bool var err error - r.options = options - for _, address := range r.Address { + request.options = options + for _, address := range request.Address { // check if the connection should be encrypted if strings.HasPrefix(address, "tls://") { shouldUseTLS = true @@ -137,13 +137,13 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if portErr != nil { return errors.Wrap(portErr, "could not parse address") } - r.addresses = append(r.addresses, addressKV{ip: addressHost, port: addressPort, tls: shouldUseTLS}) + request.addresses = append(request.addresses, addressKV{ip: addressHost, port: addressPort, tls: shouldUseTLS}) } else { - r.addresses = append(r.addresses, addressKV{ip: address, tls: shouldUseTLS}) + request.addresses = append(request.addresses, addressKV{ip: address, tls: shouldUseTLS}) } } // Pre-compile any input dsl functions before executing the request. - for _, input := range r.Inputs { + for _, input := range request.Inputs { if input.Type != "" { continue } @@ -153,11 +153,11 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } // Resolve payload paths from vars if they exists - for name, payload := range r.options.Options.VarsPayload() { + for name, payload := range request.options.Options.VarsPayload() { payloadStr, ok := payload.(string) // check if inputs contains the payload var hasPayloadName bool - for _, input := range r.Inputs { + for _, input := range request.Inputs { if input.Type != "" { continue } @@ -167,36 +167,36 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } } if ok && hasPayloadName && fileutil.FileExists(payloadStr) { - if r.Payloads == nil { - r.Payloads = make(map[string]interface{}) + if request.Payloads == nil { + request.Payloads = make(map[string]interface{}) } - r.Payloads[name] = payloadStr + request.Payloads[name] = payloadStr } } - if len(r.Payloads) > 0 { - attackType := r.AttackType + if len(request.Payloads) > 0 { + attackType := request.AttackType if attackType == "" { attackType = "batteringram" } var ok bool - r.attackType, ok = generators.StringToType[attackType] + request.attackType, ok = generators.StringToType[attackType] if !ok { return fmt.Errorf("invalid attack type provided: %s", attackType) } // Resolve payload paths if they are files. - for name, payload := range r.Payloads { + for name, payload := range request.Payloads { payloadStr, ok := payload.(string) if ok { final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) if resolveErr != nil { return errors.Wrap(resolveErr, "could not read payload file") } - r.Payloads[name] = final + request.Payloads[name] = final } } - r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath) + request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath) if err != nil { return errors.Wrap(err, "could not parse payloads") } @@ -207,19 +207,19 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if err != nil { return errors.Wrap(err, "could not get network client") } - r.dialer = client + request.dialer = client - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { - return len(r.Address) +func (request *Request) Requests() int { + return len(request.Address) } diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 75e71dda..8e0330dc 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -4,14 +4,16 @@ import ( "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "all", "": @@ -20,27 +22,27 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(itemStr))) + return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr, r.dynamicValues)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, request.dynamicValues)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { partString := extractor.Part switch partString { case "body", "all", "": @@ -62,51 +64,30 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext return nil } -// responseToDSLMap converts a DNS response to a map for use in DSL matching -func (r *Request) responseToDSLMap(req, resp, raw, host, matched string) output.InternalEvent { - data := make(output.InternalEvent, 6) - - // Some data regarding the request metadata - data["host"] = host - data["matched"] = matched - data["request"] = req - data["data"] = resp // Data is the last bytes read - data["raw"] = raw // Raw is the full transaction data for network - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath - return data +// responseToDSLMap converts a network response to a map for use in DSL matching +func (request *Request) responseToDSLMap(req, resp, raw, host, matched string) output.InternalEvent { + return output.InternalEvent{ + "host": host, + "matched": matched, + "request": req, + "data": resp, // Data is the last bytes read + "raw": raw, // Raw is the full transaction data for network + "template-id": request.options.TemplateID, + "template-info": request.options.TemplateInfo, + "template-path": request.options.TemplatePath, + } } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := r.makeResultEventItem(wrapped) - results = append(results, data) - } - return results +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index 4c8ea302..bf577d7e 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -70,8 +70,9 @@ func TestNetworkOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -84,8 +85,9 @@ func TestNetworkOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -97,8 +99,9 @@ func TestNetworkOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } @@ -189,7 +192,7 @@ func TestNetworkMakeResult(t *testing.T) { finalEvent := &output.InternalWrappedEvent{InternalEvent: event} event["ip"] = "192.168.1.1" if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract) + result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false) if ok && result != nil { finalEvent.OperatorsResult = result finalEvent.Results = request.MakeResultEvent(finalEvent) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index aad3c60e..ba996da8 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -10,11 +10,14 @@ import ( "time" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" ) @@ -22,15 +25,15 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { address, err := getAddress(input) if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "network", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "network", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not get address from url") } - for _, kv := range r.addresses { + for _, kv := range request.addresses { actualAddress := replacer.Replace(kv.ip, map[string]interface{}{"Hostname": address}) if kv.port != "" { if strings.Contains(address, ":") { @@ -39,7 +42,7 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused actualAddress = net.JoinHostPort(actualAddress, kv.port) } - if err := r.executeAddress(actualAddress, address, input, kv.tls, previous, callback); err != nil { + if err := request.executeAddress(actualAddress, address, input, kv.tls, previous, callback); err != nil { gologger.Verbose().Label("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err) continue } @@ -48,71 +51,71 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused } // executeAddress executes the request for an address -func (r *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { if !strings.Contains(actualAddress, ":") { err := errors.New("no port provided in network protocol request") - r.options.Output.Request(r.options.TemplateID, address, "network", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Progress.IncrementFailedRequestsBy(1) return err } - if r.generator != nil { - iterator := r.generator.NewIterator() + if request.generator != nil { + iterator := request.generator.NewIterator() for { value, ok := iterator.Value() if !ok { break } - if err := r.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { + if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } } } else { value := make(map[string]interface{}) - if err := r.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { + if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } } return nil } -func (r *Request) executeRequestWithPayloads(actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeRequestWithPayloads(actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { var ( hostname string conn net.Conn err error ) - r.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) + request.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil { hostname = host } if shouldUseTLS { - conn, err = r.dialer.DialTLS(context.Background(), "tcp", actualAddress) + conn, err = request.dialer.DialTLS(context.Background(), "tcp", actualAddress) } else { - conn, err = r.dialer.Dial(context.Background(), "tcp", actualAddress) + conn, err = request.dialer.Dial(context.Background(), "tcp", actualAddress) } if err != nil { - r.options.Output.Request(r.options.TemplateID, address, "network", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server request") } defer conn.Close() - _ = conn.SetReadDeadline(time.Now().Add(time.Duration(r.options.Options.Timeout) * time.Second)) + _ = conn.SetReadDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second)) - hasInteractMarkers := interactsh.HasMatchers(r.CompiledOperators) + hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators) var interactURL string - if r.options.Interactsh != nil && hasInteractMarkers { - interactURL = r.options.Interactsh.URL() + if request.options.Interactsh != nil && hasInteractMarkers { + interactURL = request.options.Interactsh.URL() } responseBuilder := &strings.Builder{} reqBuilder := &strings.Builder{} inputEvents := make(map[string]interface{}) - for _, input := range r.Inputs { + for _, input := range request.Inputs { var data []byte switch input.Type { @@ -120,32 +123,32 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin data, err = hex.DecodeString(input.Data) default: if interactURL != "" { - input.Data = r.options.Interactsh.ReplaceMarkers(input.Data, interactURL) + input.Data = request.options.Interactsh.ReplaceMarkers(input.Data, interactURL) } data = []byte(input.Data) } if err != nil { - r.options.Output.Request(r.options.TemplateID, address, "network", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } reqBuilder.Grow(len(input.Data)) finalData, dataErr := expressions.EvaluateByte(data, payloads) if dataErr != nil { - r.options.Output.Request(r.options.TemplateID, address, "network", dataErr) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, address, "network", dataErr) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } reqBuilder.Write(finalData) if varErr := expressions.ContainsUnresolvedVariables(string(finalData)); varErr != nil { - gologger.Warning().Msgf("[%s] Could not make network request for %s: %v\n", r.options.TemplateID, actualAddress, varErr) + gologger.Warning().Msgf("[%s] Could not make network request for %s: %v\n", request.options.TemplateID, actualAddress, varErr) return nil } if _, err := conn.Write(finalData); err != nil { - r.options.Output.Request(r.options.TemplateID, address, "network", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } @@ -160,44 +163,40 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin } // Run any internal extractors for the request here and add found values to map. - if r.CompiledOperators != nil { - values := r.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{input.Name: bufferStr}, r.Extract) + if request.CompiledOperators != nil { + values := request.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{input.Name: bufferStr}, request.Extract) for k, v := range values { payloads[k] = v } } } } - r.options.Progress.IncrementRequests() + request.options.Progress.IncrementRequests() - if r.options.Options.Debug || r.options.Options.DebugRequests { + if request.options.Options.Debug || request.options.Options.DebugRequests { requestOutput := reqBuilder.String() - gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", r.options.TemplateID, actualAddress) + gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", request.options.TemplateID, actualAddress) gologger.Print().Msgf("%s\nHex: %s", requestOutput, hex.EncodeToString([]byte(requestOutput))) } - r.options.Output.Request(r.options.TemplateID, actualAddress, "network", err) + request.options.Output.Request(request.options.TemplateID, actualAddress, "network", err) gologger.Verbose().Msgf("Sent TCP request to %s", actualAddress) bufferSize := 1024 - if r.ReadSize != 0 { - bufferSize = r.ReadSize + if request.ReadSize != 0 { + bufferSize = request.ReadSize } final := make([]byte, bufferSize) n, err := conn.Read(final) if err != nil && err != io.EOF { - r.options.Output.Request(r.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplateID, address, "network", err) return errors.Wrap(err, "could not read from server") } responseBuilder.Write(final[:n]) - if r.options.Options.Debug || r.options.Options.DebugResponse { - responseOutput := responseBuilder.String() - gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress) - gologger.Print().Msgf("%s\nHex: %s", responseOutput, hex.EncodeToString([]byte(responseOutput))) - } - outputEvent := r.responseToDSLMap(reqBuilder.String(), string(final[:n]), responseBuilder.String(), input, actualAddress) - outputEvent["ip"] = r.dialer.GetDialedIP(hostname) + response := responseBuilder.String() + outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress) + outputEvent["ip"] = request.dialer.GetDialedIP(hostname) for k, v := range previous { outputEvent[k] = v } @@ -208,26 +207,28 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin outputEvent[k] = v } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + var event *output.InternalWrappedEvent if interactURL == "" { - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.OperatorsResult.PayloadValues = payloads - event.Results = r.MakeResultEvent(event) - } - } + event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { + wrappedEvent.OperatorsResult.PayloadValues = payloads + }) callback(event) - } else if r.options.Interactsh != nil { - r.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ - MakeResultFunc: r.MakeResultEvent, + } else if request.options.Interactsh != nil { + event = &output.InternalWrappedEvent{InternalEvent: outputEvent} + request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ + MakeResultFunc: request.MakeResultEvent, Event: event, - Operators: r.CompiledOperators, - MatchFunc: r.Match, - ExtractFunc: r.Extract, + Operators: request.CompiledOperators, + MatchFunc: request.Match, + ExtractFunc: request.Extract, }) } + + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Network response for %s", request.options.TemplateID, actualAddress) + gologger.Print().Msgf("%s\nHex: %s", response, responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), request.options.Options.NoColor)) + } + return nil } diff --git a/v2/pkg/protocols/offlinehttp/find.go b/v2/pkg/protocols/offlinehttp/find.go index ee918ad5..2c876dab 100644 --- a/v2/pkg/protocols/offlinehttp/find.go +++ b/v2/pkg/protocols/offlinehttp/find.go @@ -12,19 +12,19 @@ import ( // getInputPaths parses the specified input paths and returns a compiled // list of finished absolute paths to the files evaluating any allowlist, denylist, // glob, file or folders, etc. -func (r *Request) getInputPaths(target string, callback func(string)) error { +func (request *Request) getInputPaths(target string, callback func(string)) error { processed := make(map[string]struct{}) // Template input includes a wildcard if strings.Contains(target, "*") { - if err := r.findGlobPathMatches(target, processed, callback); err != nil { + if err := request.findGlobPathMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find glob matches") } return nil } // Template input is either a file or a directory - file, err := r.findFileMatches(target, processed, callback) + file, err := request.findFileMatches(target, processed, callback) if err != nil { return errors.Wrap(err, "could not find file") } @@ -34,14 +34,14 @@ func (r *Request) getInputPaths(target string, callback func(string)) error { // Recursively walk down the Templates directory and run all // the template file checks - if err := r.findDirectoryMatches(target, processed, callback); err != nil { + if err := request.findDirectoryMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find directory matches") } return nil } // findGlobPathMatches returns the matched files from a glob path -func (r *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error { +func (request *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error { matches, err := filepath.Glob(absPath) if err != nil { return errors.Errorf("wildcard found, but unable to glob: %s\n", err) @@ -60,7 +60,7 @@ func (r *Request) findGlobPathMatches(absPath string, processed map[string]struc // findFileMatches finds if a path is an absolute file. If the path // is a file, it returns true otherwise false with no errors. -func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) { +func (request *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) { info, err := os.Stat(absPath) if err != nil { return false, err @@ -79,7 +79,7 @@ func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, } // findDirectoryMatches finds matches for templates from a directory -func (r *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error { +func (request *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error { err := godirwalk.Walk(absPath, &godirwalk.Options{ Unsorted: true, ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction { diff --git a/v2/pkg/protocols/offlinehttp/offlinehttp.go b/v2/pkg/protocols/offlinehttp/offlinehttp.go index c315c337..85b66932 100644 --- a/v2/pkg/protocols/offlinehttp/offlinehttp.go +++ b/v2/pkg/protocols/offlinehttp/offlinehttp.go @@ -2,6 +2,7 @@ package offlinehttp import ( "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" ) @@ -13,23 +14,23 @@ type Request struct { } // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { +func (request *Request) GetID() string { return "" } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { +func (request *Request) Compile(options *protocols.ExecuterOptions) error { for _, operator := range options.Operators { if err := operator.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.compiledOperators = append(r.compiledOperators, operator) + request.compiledOperators = append(request.compiledOperators, operator) } - r.options = options + request.options = options return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { +func (request *Request) Requests() int { return 1 } diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index ace97685..59d23470 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -6,42 +6,57 @@ import ( "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { item, ok := getMatchPart(matcher.Part, data) if !ok { - return false + return false, []string{} } switch matcher.GetType() { case matchers.StatusMatcher: - statusCode, ok := data["status_code"] + statusCode, ok := getStatusCode(data) if !ok { - return false + return false, []string{} } - return matcher.Result(matcher.MatchStatusCode(statusCode.(int))) + return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{responsehighlighter.CreateStatusCodeSnippet(data["response"].(string), statusCode)} case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(item))) + return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(item, nil)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} +} + +func getStatusCode(data map[string]interface{}) (int, bool) { + statusCodeValue, ok := data["status_code"] + if !ok { + return 0, false + } + statusCode, ok := statusCodeValue.(int) + if !ok { + return 0, false + } + return statusCode, true } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { item, ok := getMatchPart(extractor.Part, data) if !ok { return nil @@ -78,8 +93,8 @@ func getMatchPart(part string, data output.InternalEvent) (string, bool) { } // responseToDSLMap converts an HTTP response to a map for use in DSL matching -func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { - data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies())) +func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) output.InternalEvent { + data := make(output.InternalEvent, 12+len(extra)+len(resp.Header)+len(resp.Cookies())) for k, v := range extra { data[k] = v } @@ -100,41 +115,22 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r } data["all_headers"] = headers data["duration"] = duration.Seconds() - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath + data["template-id"] = request.options.TemplateID + data["template-info"] = request.options.TemplateInfo + data["template-path"] = request.options.TemplatePath return data } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := r.makeResultEventItem(wrapped) - results = append(results, data) - } - return results +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) GetCompiledOperators() []*operators.Operators { + return request.compiledOperators +} + +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index 56facee7..2f4b91fd 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -76,8 +76,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -90,8 +91,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -103,8 +105,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } @@ -201,7 +204,7 @@ func TestHTTPMakeResult(t *testing.T) { event["ip"] = "192.169.1.1" finalEvent := &output.InternalWrappedEvent{InternalEvent: event} for _, operator := range request.compiledOperators { - result, ok := operator.Execute(event, request.Match, request.Extract) + result, ok := operator.Execute(event, request.Match, request.Extract, false) if ok && result != nil { finalEvent.OperatorsResult = result finalEvent.Results = request.MakeResultEvent(finalEvent) diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 27b10e9d..1eb4bfc5 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -8,11 +8,13 @@ import ( "strings" "github.com/pkg/errors" + "github.com/remeh/sizedwaitgroup" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" - "github.com/remeh/sizedwaitgroup" ) var _ protocols.Request = &Request{} @@ -20,10 +22,10 @@ var _ protocols.Request = &Request{} const maxSize = 5 * 1024 * 1024 // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - wg := sizedwaitgroup.New(r.options.Options.BulkSize) +func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + wg := sizedwaitgroup.New(request.options.Options.BulkSize) - err := r.getInputPaths(input, func(data string) { + err := request.getInputPaths(input, func(data string) { wg.Add() go func(data string) { @@ -59,11 +61,11 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused return } - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped offline-http request for %s", r.options.TemplateID, data) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped offline-http request for %s", request.options.TemplateID, data) gologger.Print().Msgf("%s", dataStr) } - gologger.Verbose().Msgf("[%s] Sent OFFLINE-HTTP request to %s", r.options.TemplateID, data) + gologger.Verbose().Msgf("[%s] Sent OFFLINE-HTTP request to %s", request.options.TemplateID, data) dumpedResponse, err := httputil.DumpResponse(resp, true) if err != nil { @@ -77,31 +79,23 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused return } - outputEvent := r.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), headersToString(resp.Header), 0, nil) + outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), headersToString(resp.Header), 0, nil) outputEvent["ip"] = "" for k, v := range previous { outputEvent[k] = v } - for _, operator := range r.compiledOperators { - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - var ok bool - - event.OperatorsResult, ok = operator.Execute(outputEvent, r.Match, r.Extract) - if ok && event.OperatorsResult != nil { - event.Results = r.MakeResultEvent(event) - } - callback(event) - } + event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) + callback(event) }(data) }) wg.Wait() if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "file", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "file", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } - r.options.Progress.IncrementRequests() + request.options.Progress.IncrementRequests() return nil } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 9ca38333..d77ba147 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -74,13 +74,49 @@ type Request interface { // condition matching. So, two requests can be sent and their match can // be evaluated from the third request by using the IDs for both requests. GetID() string - // Match performs matching operation for a matcher on model and returns true or false. - Match(data map[string]interface{}, matcher *matchers.Matcher) bool + // Match performs matching operation for a matcher on model and returns: + // true and a list of matched snippets if the matcher type is supports it + // otherwise false and an empty string slice + Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) // Extract performs extracting operation for an extractor on model and returns true or false. Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error + // MakeResultEventItem creates a result event from internal wrapped event. Intended to be used by MakeResultEventItem internally + MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent + // MakeResultEvent creates a flat list of result events from an internal wrapped event, based on successful matchers and extracted data + MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent + // GetCompiledOperators returns a list of the compiled operators + GetCompiledOperators() []*operators.Operators } // OutputEventCallback is a callback event for any results found during scanning. type OutputEventCallback func(result *output.InternalWrappedEvent) + +func MakeDefaultResultEvent(request Request, wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + if len(wrapped.OperatorsResult.DynamicValues) > 0 && !wrapped.OperatorsResult.Matched { + return nil + } + + results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) + + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for matcherNames := range wrapped.OperatorsResult.Matches { + data := request.MakeResultEventItem(wrapped) + data.MatcherName = matcherNames + results = append(results, data) + } + } else if len(wrapped.OperatorsResult.Extracts) > 0 { + for k, v := range wrapped.OperatorsResult.Extracts { + data := request.MakeResultEventItem(wrapped) + data.ExtractorName = k + data.ExtractedResults = v + results = append(results, data) + } + } else { + data := request.MakeResultEventItem(wrapped) + results = append(results, data) + } + return results +} diff --git a/v2/pkg/workflows/execute_test.go b/v2/pkg/workflows/execute_test.go index 26f7c2b9..6d9ab6a0 100644 --- a/v2/pkg/workflows/execute_test.go +++ b/v2/pkg/workflows/execute_test.go @@ -3,12 +3,13 @@ package workflows import ( "testing" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/stretchr/testify/require" ) func TestWorkflowsSimple(t *testing.T) { @@ -106,7 +107,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ {OperatorsResult: &operators.Result{ - Matches: map[string]struct{}{"tomcat": {}}, + Matches: map[string][]string{"tomcat": {}}, Extracts: map[string][]string{}, }}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, @@ -134,7 +135,7 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ {OperatorsResult: &operators.Result{ - Matches: map[string]struct{}{"tomcat": {}}, + Matches: map[string][]string{"tomcat": {}}, Extracts: map[string][]string{}, }}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}},