diff --git a/v2/pkg/executer/executer_dns.go b/v2/pkg/executer/executer_dns.go index 86d5b4e0..2b7bd8ca 100644 --- a/v2/pkg/executer/executer_dns.go +++ b/v2/pkg/executer/executer_dns.go @@ -8,7 +8,6 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/pkg/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/requests" "github.com/projectdiscovery/nuclei/v2/pkg/templates" ) @@ -95,45 +94,6 @@ func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, reqURL string) *Result { fmt.Fprintf(os.Stderr, "%s\n", resp.String()) } - matcherCondition := e.dnsRequest.GetMatchersCondition() - - for _, matcher := range e.dnsRequest.Matchers { - // Check if the matcher matched - if !matcher.MatchDNS(resp) { - // If the condition is AND we haven't matched, return. - if matcherCondition == matchers.ANDCondition { - return result - } - } else { - // If the matcher has matched, and its an OR - // write the first output then move to next matcher. - if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 { - e.writeOutputDNS(domain, compiledRequest, resp, matcher, nil) - result.GotResults = true - } - } - } - - // All matchers have successfully completed so now start with the - // next task which is extraction of input from matchers. - var extractorResults []string - - for _, extractor := range e.dnsRequest.Extractors { - for match := range extractor.ExtractDNS(resp) { - if !extractor.Internal { - extractorResults = append(extractorResults, match) - } - } - } - - // Write a final string of output if matcher type is - // AND or if we have extractors for the mechanism too. - if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition { - e.writeOutputDNS(domain, compiledRequest, resp, nil, extractorResults) - - result.GotResults = true - } - return result } diff --git a/v2/pkg/operators/extractors/extract.go b/v2/pkg/operators/extractors/extract.go index 0269aaa3..31c9aa6f 100644 --- a/v2/pkg/operators/extractors/extract.go +++ b/v2/pkg/operators/extractors/extract.go @@ -2,25 +2,8 @@ package extractors import "github.com/projectdiscovery/nuclei/v2/pkg/types" -// Extract extracts data from an output structure based on user options -func (e *Extractor) Extract(data map[string]interface{}) map[string]struct{} { - part, ok := data[e.Part] - if !ok { - return nil - } - partString := types.ToString(part) - - switch e.extractorType { - case RegexExtractor: - return e.extractRegex(partString) - case KValExtractor: - return e.extractKVal(data) - } - return nil -} - -// extractRegex extracts text from a corpus and returns it -func (e *Extractor) extractRegex(corpus string) map[string]struct{} { +// ExtractRegex extracts text from a corpus and returns it +func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} { results := make(map[string]struct{}) groupPlusOne := e.RegexGroup + 1 @@ -41,8 +24,8 @@ func (e *Extractor) extractRegex(corpus string) map[string]struct{} { return results } -// extractKVal extracts key value pairs from a data map -func (e *Extractor) extractKVal(data map[string]interface{}) map[string]struct{} { +// ExtractKval extracts key value pairs from a data map +func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{} { results := make(map[string]struct{}) for _, k := range e.KVal { diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index f800e6a0..f593a174 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -44,3 +44,8 @@ var ExtractorTypes = map[string]ExtractorType{ "regex": RegexExtractor, "kval": KValExtractor, } + +// GetType returns the type of the matcher +func (e *Extractor) GetType() ExtractorType { + return e.extractorType +} diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 19ba7d29..2b31386c 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -5,37 +5,8 @@ import ( "strings" ) -// Match matches a generic data response again a given matcher -func (m *Matcher) Match(data map[string]interface{}) bool { - part, ok := data[m.Part] - if !ok { - return false - } - partString := part.(string) - - switch m.matcherType { - case StatusMatcher: - statusCode, ok := data["status_code"] - if !ok { - return false - } - return m.isNegative(m.matchStatusCode(statusCode.(int))) - case SizeMatcher: - return m.isNegative(m.matchSizeCode(len(partString))) - case WordsMatcher: - return m.isNegative(m.matchWords(partString)) - case RegexMatcher: - return m.isNegative(m.matchRegex(partString)) - case BinaryMatcher: - return m.isNegative(m.matchBinary(partString)) - case DSLMatcher: - return m.isNegative(m.matchDSL(data)) - } - return false -} - -// matchStatusCode matches a status code check against an HTTP Response -func (m *Matcher) matchStatusCode(statusCode int) bool { +// MatchStatusCode matches a status code check against a corpus +func (m *Matcher) MatchStatusCode(statusCode int) bool { // Iterate over all the status codes accepted as valid // // Status codes don't support AND conditions. @@ -50,8 +21,8 @@ func (m *Matcher) matchStatusCode(statusCode int) bool { return false } -// matchStatusCode matches a size check against an HTTP Response -func (m *Matcher) matchSizeCode(length int) bool { +// MatchSize matches a size check against a corpus +func (m *Matcher) MatchSize(length int) bool { // Iterate over all the sizes accepted as valid // // Sizes codes don't support AND conditions. @@ -66,8 +37,8 @@ func (m *Matcher) matchSizeCode(length int) bool { return false } -// matchWords matches a word check against an HTTP Response/Headers. -func (m *Matcher) matchWords(corpus string) bool { +// MatchWords matches a word check against a corpus. +func (m *Matcher) MatchWords(corpus string) bool { // Iterate over all the words accepted as valid for i, word := range m.Words { // Continue if the word doesn't match @@ -94,8 +65,8 @@ func (m *Matcher) matchWords(corpus string) bool { return false } -// matchRegex matches a regex check against an HTTP Response/Headers. -func (m *Matcher) matchRegex(corpus string) bool { +// MatchRegex matches a regex check against a corpus +func (m *Matcher) MatchRegex(corpus string) bool { // Iterate over all the regexes accepted as valid for i, regex := range m.regexCompiled { // Continue if the regex doesn't match @@ -122,8 +93,8 @@ func (m *Matcher) matchRegex(corpus string) bool { return false } -// matchWords matches a word check against an HTTP Response/Headers. -func (m *Matcher) matchBinary(corpus string) bool { +// MatchBinary matches a binary check against a corpus +func (m *Matcher) MatchBinary(corpus string) bool { // Iterate over all the words accepted as valid for i, binary := range m.Binary { // Continue if the word doesn't match @@ -151,11 +122,11 @@ func (m *Matcher) matchBinary(corpus string) bool { return false } -// matchDSL matches on a generic map result -func (m *Matcher) matchDSL(mp map[string]interface{}) bool { +// MatchDSL matches on a generic map result +func (m *Matcher) MatchDSL(data map[string]interface{}) bool { // Iterate over all the expressions accepted as valid for i, expression := range m.dslCompiled { - result, err := expression.Evaluate(mp) + result, err := expression.Evaluate(data) if err != nil { continue } diff --git a/v2/pkg/operators/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go index 7d02ecf9..2ab27c40 100644 --- a/v2/pkg/operators/matchers/match_test.go +++ b/v2/pkg/operators/matchers/match_test.go @@ -9,22 +9,22 @@ import ( func TestANDCondition(t *testing.T) { m := &Matcher{condition: ANDCondition, Words: []string{"a", "b"}} - matched := m.matchWords("a b") + matched := m.MatchWords("a b") require.True(t, matched, "Could not match valid AND condition") - matched = m.matchWords("b") + matched = m.MatchWords("b") require.False(t, matched, "Could match invalid AND condition") } func TestORCondition(t *testing.T) { m := &Matcher{condition: ORCondition, Words: []string{"a", "b"}} - matched := m.matchWords("a b") + matched := m.MatchWords("a b") require.True(t, matched, "Could not match valid OR condition") - matched = m.matchWords("b") + matched = m.MatchWords("b") require.True(t, matched, "Could not match valid OR condition") - matched = m.matchWords("c") + matched = m.MatchWords("c") require.False(t, matched, "Could match invalid OR condition") } diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index 930461be..20297587 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -88,11 +88,15 @@ var ConditionTypes = map[string]ConditionType{ "or": ORCondition, } -// isNegative reverts the results of the match if the matcher -// is of type negative. -func (m *Matcher) isNegative(data bool) bool { +// Result reverts the results of the match if the matcher is of type negative. +func (m *Matcher) Result(data bool) bool { if m.Negative { return !data } return data } + +// GetType returns the type of the matcher +func (m *Matcher) GetType() MatcherType { + return m.matcherType +} diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 1c983cff..39cdce26 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -3,6 +3,7 @@ package operators import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" ) // Operators contains the operators that can be applied on protocols @@ -31,12 +32,20 @@ type Result struct { Matches map[string]struct{} // Extracts contains all the data extracted from inputs Extracts map[string][]string + // OutputExtracts is the list of extracts to be displayed on screen. + OutputExtracts []string // DynamicValues contains any dynamic values to be templated DynamicValues map[string]string } +// MatchFunc performs matching operation for a matcher on model and returns true or false. +type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) bool + +// ExtractFunc performs extracting operation for a extractor on model and returns true or false. +type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} + // Execute executes the operators on data and returns a result structure -func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) { +func (r *Operators) Execute(data output.Event, match MatchFunc, extract ExtractFunc) (*Result, bool) { matcherCondition := r.GetMatchersCondition() result := &Result{ @@ -46,7 +55,7 @@ func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) { } for _, matcher := range r.Matchers { // Check if the matcher matched - if !matcher.Match(data) { + if !match(data, matcher) { // If the condition is AND we haven't matched, try next request. if matcherCondition == matchers.ANDCondition { return nil, false @@ -62,9 +71,10 @@ func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) { // All matchers have successfully completed so now start with the // next task which is extraction of input from matchers. - var extractorResults, outputExtractorResults []string for _, extractor := range r.Extractors { - for match := range extractor.Extract(data) { + var extractorResults []string + + for match := range extract(data, extractor) { extractorResults = append(extractorResults, match) if extractor.Internal { @@ -72,7 +82,7 @@ func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) { result.DynamicValues[extractor.Name] = match } } else { - outputExtractorResults = append(outputExtractorResults, match) + result.OutputExtracts = append(result.OutputExtracts, match) } } result.Extracts[extractor.Name] = extractorResults diff --git a/v2/pkg/protocols/http/matchers.go b/v2/pkg/protocols/http/matchers.go index 38d6fb39..185f0c40 100644 --- a/v2/pkg/protocols/http/matchers.go +++ b/v2/pkg/protocols/http/matchers.go @@ -5,8 +5,69 @@ import ( "net/http/httputil" "strings" "time" + + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" ) +// Match matches a generic data response again a given matcher +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { + part, ok := data[matcher.Part] + if !ok { + return false + } + partString := part.(string) + + switch partString { + case "header": + partString = "all_headers" + case "all": + partString = "raw" + } + switch matcher.GetType() { + case matchers.StatusMatcher: + statusCode, ok := data["status_code"] + if !ok { + return false + } + return matcher.Result(matcher.MatchStatusCode(statusCode.(int))) + case matchers.SizeMatcher: + return matcher.Result(matcher.MatchSize(len(partString))) + case matchers.WordsMatcher: + return matcher.Result(matcher.MatchWords(partString)) + case matchers.RegexMatcher: + return matcher.Result(matcher.MatchRegex(partString)) + case matchers.BinaryMatcher: + return matcher.Result(matcher.MatchBinary(partString)) + case matchers.DSLMatcher: + return matcher.Result(matcher.MatchDSL(data)) + } + return false +} + +// Extract performs extracting operation for a extractor on model and returns true or false. +func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { + part, ok := data[extractor.Part] + if !ok { + return nil + } + partString := part.(string) + + switch partString { + case "header": + partString = "all_headers" + case "all": + partString = "raw" + } + switch extractor.GetType() { + case extractors.RegexExtractor: + return extractor.ExtractRegex(partString) + case extractors.KValExtractor: + return extractor.ExtractKval(data) + } + return nil +} + // responseToDSLMap converts a HTTP response to a map for use in DSL matching func responseToDSLMap(resp *http.Response, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { data := make(map[string]interface{}, len(extra)+6+len(resp.Header)+len(resp.Cookies())) @@ -25,13 +86,11 @@ func responseToDSLMap(resp *http.Response, body, headers string, duration time.D k = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(k, "-", "_"))) data[k] = strings.Join(v, " ") } - data["header"] = headers data["all_headers"] = headers if r, err := httputil.DumpResponse(resp, true); err == nil { rawString := string(r) data["raw"] = rawString - data["all"] = rawString } data["duration"] = duration.Seconds() return data diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 09542784..d6fbf5b3 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -1,6 +1,8 @@ package protocols import ( + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/types" "go.uber.org/ratelimit" @@ -12,6 +14,10 @@ type Executer interface { Compile(options ExecuterOptions) error // Requests returns the total number of requests the rule will perform Requests() int64 + // Match performs matching operation for a matcher on model and returns true or false. + Match(data map[string]interface{}, matcher *matchers.Matcher) bool + // Extract performs extracting operation for a extractor on model and returns true or false. + Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // Execute executes the protocol requests and returns an output event channel. Execute(input string) (bool, error) // ExecuteWithResults executes the protocol requests and returns results instead of writing them.