More refactoring of nuclei packages

v2/pkg/executer/executer_dns.go

@@ -8,7 +8,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/internal/progress"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
 )
@@ -95,45 +94,6 @@ func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, reqURL string) *Result {
 		fmt.Fprintf(os.Stderr, "%s\n", resp.String())
 	}
 
-	matcherCondition := e.dnsRequest.GetMatchersCondition()
-
-	for _, matcher := range e.dnsRequest.Matchers {
-		// Check if the matcher matched
-		if !matcher.MatchDNS(resp) {
-			// If the condition is AND we haven't matched, return.
-			if matcherCondition == matchers.ANDCondition {
-				return result
-			}
-		} else {
-			// If the matcher has matched, and its an OR
-			// write the first output then move to next matcher.
-			if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 {
-				e.writeOutputDNS(domain, compiledRequest, resp, matcher, nil)
-				result.GotResults = true
-			}
-		}
-	}
-
-	// All matchers have successfully completed so now start with the
-	// next task which is extraction of input from matchers.
-	var extractorResults []string
-
-	for _, extractor := range e.dnsRequest.Extractors {
-		for match := range extractor.ExtractDNS(resp) {
-			if !extractor.Internal {
-				extractorResults = append(extractorResults, match)
-			}
-		}
-	}
-
-	// Write a final string of output if matcher type is
-	// AND or if we have extractors for the mechanism too.
-	if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
-		e.writeOutputDNS(domain, compiledRequest, resp, nil, extractorResults)
-
-		result.GotResults = true
-	}
-
 	return result
 }
 

v2/pkg/operators/extractors/extract.go

@@ -2,25 +2,8 @@ package extractors
 
 import "github.com/projectdiscovery/nuclei/v2/pkg/types"
 
-// Extract extracts data from an output structure based on user options
+// ExtractRegex extracts text from a corpus and returns it
-func (e *Extractor) Extract(data map[string]interface{}) map[string]struct{} {
+func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} {
-	part, ok := data[e.Part]
-	if !ok {
-		return nil
-	}
-	partString := types.ToString(part)
-
-	switch e.extractorType {
-	case RegexExtractor:
-		return e.extractRegex(partString)
-	case KValExtractor:
-		return e.extractKVal(data)
-	}
-	return nil
-}
-
-// extractRegex extracts text from a corpus and returns it
-func (e *Extractor) extractRegex(corpus string) map[string]struct{} {
 	results := make(map[string]struct{})
 
 	groupPlusOne := e.RegexGroup + 1
@@ -41,8 +24,8 @@ func (e *Extractor) extractRegex(corpus string) map[string]struct{} {
 	return results
 }
 
-// extractKVal extracts key value pairs from a data map
+// ExtractKval extracts key value pairs from a data map
-func (e *Extractor) extractKVal(data map[string]interface{}) map[string]struct{} {
+func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{} {
 	results := make(map[string]struct{})
 
 	for _, k := range e.KVal {

v2/pkg/operators/extractors/extractors.go

@@ -44,3 +44,8 @@ var ExtractorTypes = map[string]ExtractorType{
 	"regex": RegexExtractor,
 	"kval":  KValExtractor,
 }
+
+// GetType returns the type of the matcher
+func (e *Extractor) GetType() ExtractorType {
+	return e.extractorType
+}

v2/pkg/operators/matchers/match.go

@@ -5,37 +5,8 @@ import (
 	"strings"
 )
 
-// Match matches a generic data response again a given matcher
+// MatchStatusCode matches a status code check against a corpus
-func (m *Matcher) Match(data map[string]interface{}) bool {
+func (m *Matcher) MatchStatusCode(statusCode int) bool {
-	part, ok := data[m.Part]
-	if !ok {
-		return false
-	}
-	partString := part.(string)
-
-	switch m.matcherType {
-	case StatusMatcher:
-		statusCode, ok := data["status_code"]
-		if !ok {
-			return false
-		}
-		return m.isNegative(m.matchStatusCode(statusCode.(int)))
-	case SizeMatcher:
-		return m.isNegative(m.matchSizeCode(len(partString)))
-	case WordsMatcher:
-		return m.isNegative(m.matchWords(partString))
-	case RegexMatcher:
-		return m.isNegative(m.matchRegex(partString))
-	case BinaryMatcher:
-		return m.isNegative(m.matchBinary(partString))
-	case DSLMatcher:
-		return m.isNegative(m.matchDSL(data))
-	}
-	return false
-}
-
-// matchStatusCode matches a status code check against an HTTP Response
-func (m *Matcher) matchStatusCode(statusCode int) bool {
 	// Iterate over all the status codes accepted as valid
 	//
 	// Status codes don't support AND conditions.
@@ -50,8 +21,8 @@ func (m *Matcher) matchStatusCode(statusCode int) bool {
 	return false
 }
 
-// matchStatusCode matches a size check against an HTTP Response
+// MatchSize matches a size check against a corpus
-func (m *Matcher) matchSizeCode(length int) bool {
+func (m *Matcher) MatchSize(length int) bool {
 	// Iterate over all the sizes accepted as valid
 	//
 	// Sizes codes don't support AND conditions.
@@ -66,8 +37,8 @@ func (m *Matcher) matchSizeCode(length int) bool {
 	return false
 }
 
-// matchWords matches a word check against an HTTP Response/Headers.
+// MatchWords matches a word check against a corpus.
-func (m *Matcher) matchWords(corpus string) bool {
+func (m *Matcher) MatchWords(corpus string) bool {
 	// Iterate over all the words accepted as valid
 	for i, word := range m.Words {
 		// Continue if the word doesn't match
@@ -94,8 +65,8 @@ func (m *Matcher) matchWords(corpus string) bool {
 	return false
 }
 
-// matchRegex matches a regex check against an HTTP Response/Headers.
+// MatchRegex matches a regex check against a corpus
-func (m *Matcher) matchRegex(corpus string) bool {
+func (m *Matcher) MatchRegex(corpus string) bool {
 	// Iterate over all the regexes accepted as valid
 	for i, regex := range m.regexCompiled {
 		// Continue if the regex doesn't match
@@ -122,8 +93,8 @@ func (m *Matcher) matchRegex(corpus string) bool {
 	return false
 }
 
-// matchWords matches a word check against an HTTP Response/Headers.
+// MatchBinary matches a binary check against a corpus
-func (m *Matcher) matchBinary(corpus string) bool {
+func (m *Matcher) MatchBinary(corpus string) bool {
 	// Iterate over all the words accepted as valid
 	for i, binary := range m.Binary {
 		// Continue if the word doesn't match
@@ -151,11 +122,11 @@ func (m *Matcher) matchBinary(corpus string) bool {
 	return false
 }
 
-// matchDSL matches on a generic map result
+// MatchDSL matches on a generic map result
-func (m *Matcher) matchDSL(mp map[string]interface{}) bool {
+func (m *Matcher) MatchDSL(data map[string]interface{}) bool {
 	// Iterate over all the expressions accepted as valid
 	for i, expression := range m.dslCompiled {
-		result, err := expression.Evaluate(mp)
+		result, err := expression.Evaluate(data)
 		if err != nil {
 			continue
 		}

v2/pkg/operators/matchers/match_test.go

@@ -9,22 +9,22 @@ import (
 func TestANDCondition(t *testing.T) {
 	m := &Matcher{condition: ANDCondition, Words: []string{"a", "b"}}
 
-	matched := m.matchWords("a b")
+	matched := m.MatchWords("a b")
 	require.True(t, matched, "Could not match valid AND condition")
 
-	matched = m.matchWords("b")
+	matched = m.MatchWords("b")
 	require.False(t, matched, "Could match invalid AND condition")
 }
 
 func TestORCondition(t *testing.T) {
 	m := &Matcher{condition: ORCondition, Words: []string{"a", "b"}}
 
-	matched := m.matchWords("a b")
+	matched := m.MatchWords("a b")
 	require.True(t, matched, "Could not match valid OR condition")
 
-	matched = m.matchWords("b")
+	matched = m.MatchWords("b")
 	require.True(t, matched, "Could not match valid OR condition")
 
-	matched = m.matchWords("c")
+	matched = m.MatchWords("c")
 	require.False(t, matched, "Could match invalid OR condition")
 }

v2/pkg/operators/matchers/matchers.go

@@ -88,11 +88,15 @@ var ConditionTypes = map[string]ConditionType{
 	"or":  ORCondition,
 }
 
-// isNegative reverts the results of the match if the matcher
+// Result reverts the results of the match if the matcher is of type negative.
-// is of type negative.
+func (m *Matcher) Result(data bool) bool {
-func (m *Matcher) isNegative(data bool) bool {
 	if m.Negative {
 		return !data
 	}
 	return data
 }
+
+// GetType returns the type of the matcher
+func (m *Matcher) GetType() MatcherType {
+	return m.matcherType
+}

v2/pkg/operators/operators.go

@@ -3,6 +3,7 @@ package operators
 import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 )
 
 // Operators contains the operators that can be applied on protocols
@@ -31,12 +32,20 @@ type Result struct {
 	Matches map[string]struct{}
 	// Extracts contains all the data extracted from inputs
 	Extracts map[string][]string
+	// OutputExtracts is the list of extracts to be displayed on screen.
+	OutputExtracts []string
 	// DynamicValues contains any dynamic values to be templated
 	DynamicValues map[string]string
 }
 
+// MatchFunc performs matching operation for a matcher on model and returns true or false.
+type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) bool
+
+// ExtractFunc performs extracting operation for a extractor on model and returns true or false.
+type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
+
 // Execute executes the operators on data and returns a result structure
-func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) {
+func (r *Operators) Execute(data output.Event, match MatchFunc, extract ExtractFunc) (*Result, bool) {
 	matcherCondition := r.GetMatchersCondition()
 
 	result := &Result{
@@ -46,7 +55,7 @@ func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) {
 	}
 	for _, matcher := range r.Matchers {
 		// Check if the matcher matched
-		if !matcher.Match(data) {
+		if !match(data, matcher) {
 			// If the condition is AND we haven't matched, try next request.
 			if matcherCondition == matchers.ANDCondition {
 				return nil, false
@@ -62,9 +71,10 @@ func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) {
 
 	// All matchers have successfully completed so now start with the
 	// next task which is extraction of input from matchers.
-	var extractorResults, outputExtractorResults []string
 	for _, extractor := range r.Extractors {
-		for match := range extractor.Extract(data) {
+		var extractorResults []string
+
+		for match := range extract(data, extractor) {
 			extractorResults = append(extractorResults, match)
 
 			if extractor.Internal {
@@ -72,7 +82,7 @@ func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) {
 					result.DynamicValues[extractor.Name] = match
 				}
 			} else {
-				outputExtractorResults = append(outputExtractorResults, match)
+				result.OutputExtracts = append(result.OutputExtracts, match)
 			}
 		}
 		result.Extracts[extractor.Name] = extractorResults

v2/pkg/protocols/http/matchers.go

@@ -5,8 +5,69 @@ import (
 	"net/http/httputil"
 	"strings"
 	"time"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 )
 
+// Match matches a generic data response again a given matcher
+func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
+	part, ok := data[matcher.Part]
+	if !ok {
+		return false
+	}
+	partString := part.(string)
+
+	switch partString {
+	case "header":
+		partString = "all_headers"
+	case "all":
+		partString = "raw"
+	}
+	switch matcher.GetType() {
+	case matchers.StatusMatcher:
+		statusCode, ok := data["status_code"]
+		if !ok {
+			return false
+		}
+		return matcher.Result(matcher.MatchStatusCode(statusCode.(int)))
+	case matchers.SizeMatcher:
+		return matcher.Result(matcher.MatchSize(len(partString)))
+	case matchers.WordsMatcher:
+		return matcher.Result(matcher.MatchWords(partString))
+	case matchers.RegexMatcher:
+		return matcher.Result(matcher.MatchRegex(partString))
+	case matchers.BinaryMatcher:
+		return matcher.Result(matcher.MatchBinary(partString))
+	case matchers.DSLMatcher:
+		return matcher.Result(matcher.MatchDSL(data))
+	}
+	return false
+}
+
+// Extract performs extracting operation for a extractor on model and returns true or false.
+func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
+	part, ok := data[extractor.Part]
+	if !ok {
+		return nil
+	}
+	partString := part.(string)
+
+	switch partString {
+	case "header":
+		partString = "all_headers"
+	case "all":
+		partString = "raw"
+	}
+	switch extractor.GetType() {
+	case extractors.RegexExtractor:
+		return extractor.ExtractRegex(partString)
+	case extractors.KValExtractor:
+		return extractor.ExtractKval(data)
+	}
+	return nil
+}
+
 // responseToDSLMap converts a HTTP response to a map for use in DSL matching
 func responseToDSLMap(resp *http.Response, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} {
 	data := make(map[string]interface{}, len(extra)+6+len(resp.Header)+len(resp.Cookies()))
@@ -25,13 +86,11 @@ func responseToDSLMap(resp *http.Response, body, headers string, duration time.D
 		k = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(k, "-", "_")))
 		data[k] = strings.Join(v, " ")
 	}
-	data["header"] = headers
 	data["all_headers"] = headers
 
 	if r, err := httputil.DumpResponse(resp, true); err == nil {
 		rawString := string(r)
 		data["raw"] = rawString
-		data["all"] = rawString
 	}
 	data["duration"] = duration.Seconds()
 	return data

v2/pkg/protocols/protocols.go

@@ -1,6 +1,8 @@
 package protocols
 
 import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"go.uber.org/ratelimit"
@@ -12,6 +14,10 @@ type Executer interface {
 	Compile(options ExecuterOptions) error
 	// Requests returns the total number of requests the rule will perform
 	Requests() int64
+	// Match performs matching operation for a matcher on model and returns true or false.
+	Match(data map[string]interface{}, matcher *matchers.Matcher) bool
+	// Extract performs extracting operation for a extractor on model and returns true or false.
+	Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
 	// Execute executes the protocol requests and returns an output event channel.
 	Execute(input string) (bool, error)
 	// ExecuteWithResults executes the protocol requests and returns results instead of writing them.