Merge pull request #45 from projectdiscovery/feature-dsl-query

Complex Matcher Query

go.mod

@@ -3,6 +3,7 @@ module github.com/projectdiscovery/nuclei
 go 1.14
 
 require (
+	github.com/Knetic/govaluate v3.0.0+incompatible
 	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
 	github.com/miekg/dns v1.1.29
 	github.com/pkg/errors v0.9.1

go.sum

@@ -1,3 +1,6 @@
+github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4=
+github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
+github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

pkg/matchers/compile.go

@@ -1,8 +1,15 @@
 package matchers
 
 import (
+	"crypto/md5"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
 	"fmt"
 	"regexp"
+	"strings"
+
+	"github.com/Knetic/govaluate"
 )
 
 // CompileMatchers performs the initial setup operation on a matcher
@@ -25,6 +32,16 @@ func (m *Matcher) CompileMatchers() error {
 		m.regexCompiled = append(m.regexCompiled, compiled)
 	}
 
+	// Compile the dsl expressions
+	for _, dsl := range m.DSL {
+		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(dsl, helperFunctions())
+		if err != nil {
+			return fmt.Errorf("could not compile dsl: %s", dsl)
+		}
+
+		m.dslCompiled = append(m.dslCompiled, compiled)
+	}
+
 	// Setup the condition type, if any.
 	if m.Condition != "" {
 		m.condition, ok = ConditionTypes[m.Condition]
@@ -46,3 +63,69 @@ func (m *Matcher) CompileMatchers() error {
 	}
 	return nil
 }
+
+func helperFunctions() (functions map[string]govaluate.ExpressionFunction) {
+	functions = make(map[string]govaluate.ExpressionFunction)
+	// strings
+	functions["len"] = func(args ...interface{}) (interface{}, error) {
+		length := len(args[0].(string))
+		return (float64)(length), nil
+	}
+	functions["toupper"] = func(args ...interface{}) (interface{}, error) {
+		return strings.ToUpper(args[0].(string)), nil
+	}
+	functions["tolower"] = func(args ...interface{}) (interface{}, error) {
+		return strings.ToLower(args[0].(string)), nil
+	}
+	functions["replace"] = func(args ...interface{}) (interface{}, error) {
+		return strings.Replace(args[0].(string), args[1].(string), args[2].(string), -1), nil
+	}
+	functions["trim"] = func(args ...interface{}) (interface{}, error) {
+		return strings.Trim(args[0].(string), args[2].(string)), nil
+	}
+	functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
+		return strings.TrimLeft(args[0].(string), args[1].(string)), nil
+	}
+	functions["trimright"] = func(args ...interface{}) (interface{}, error) {
+		return strings.TrimRight(args[0].(string), args[1].(string)), nil
+	}
+	functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
+		return strings.TrimSpace(args[0].(string)), nil
+	}
+	functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
+		return strings.TrimPrefix(args[0].(string), args[1].(string)), nil
+	}
+	functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
+		return strings.TrimSuffix(args[0].(string), args[1].(string)), nil
+	}
+	// encoding
+	functions["base64"] = func(args ...interface{}) (interface{}, error) {
+		sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
+		return sEnc, nil
+	}
+	functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
+		sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
+		return sEnc, nil
+	}
+	// hashing
+	functions["md5"] = func(args ...interface{}) (interface{}, error) {
+		hash := md5.Sum([]byte(args[0].(string)))
+		return hex.EncodeToString(hash[:]), nil
+	}
+	functions["sha256"] = func(args ...interface{}) (interface{}, error) {
+		return sha256.Sum256([]byte(args[0].(string))), nil
+	}
+	// search
+	functions["contains"] = func(args ...interface{}) (interface{}, error) {
+		return strings.Contains(args[0].(string), args[1].(string)), nil
+	}
+	functions["regex"] = func(args ...interface{}) (interface{}, error) {
+		compiled, err := regexp.Compile(args[0].(string))
+		if err != nil {
+			return nil, err
+		}
+		return compiled.MatchString(args[1].(string)), nil
+	}
+
+	return
+}

pkg/matchers/match.go

@@ -51,6 +51,9 @@ func (m *Matcher) Match(resp *http.Response, body, headers string) bool {
 			}
 			return m.matchBinary(body)
 		}
+	case DSLMatcher:
+		// Match complex query
+		return m.matchDSL(httpToMap(resp, body, headers))
 	}
 	return false
 }
@@ -70,6 +73,9 @@ func (m *Matcher) MatchDNS(msg *dns.Msg) bool {
 	case BinaryMatcher:
 		// Match binary characters check
 		return m.matchBinary(msg.String())
+	case DSLMatcher:
+		// Match complex query
+		return m.matchDSL(dnsToMap(msg))
 	}
 	return false
 }
@@ -192,3 +198,38 @@ func (m *Matcher) matchBinary(corpus string) bool {
 	}
 	return false
 }
+
+// matchDSL matches on a generic map result
+func (m *Matcher) matchDSL(mp map[string]interface{}) bool {
+	// Iterate over all the regexes accepted as valid
+	for i, expression := range m.dslCompiled {
+		result, err := expression.Evaluate(mp)
+		if err != nil {
+			continue
+		}
+		var bResult bool
+		bResult, ok := result.(bool)
+
+		// Continue if the regex doesn't match
+		if !ok || !bResult {
+			// If we are in an AND request and a match failed,
+			// return false as the AND condition fails on any single mismatch.
+			if m.condition == ANDCondition {
+				return false
+			}
+			// Continue with the flow since its an OR Condition.
+			continue
+		}
+
+		// If the condition was an OR, return on the first match.
+		if m.condition == ORCondition {
+			return true
+		}
+
+		// If we are at the end of the dsl, return with true
+		if len(m.dslCompiled)-1 == i {
+			return true
+		}
+	}
+	return false
+}

pkg/matchers/matchers.go

@@ -2,6 +2,8 @@ package matchers
 
 import (
 	"regexp"
+
+	"github.com/Knetic/govaluate"
 )
 
 // Matcher is used to identify whether a template was successful.
@@ -25,6 +27,10 @@ type Matcher struct {
 	regexCompiled []*regexp.Regexp
 	// Binary are the binary characters required to be present in the response
 	Binary []string `yaml:"binary,omitempty"`
+	// DSL are the dsl queries
+	DSL []string `yaml:"dsl,omitempty"`
+	// dslCompiled is the compiled variant
+	dslCompiled []*govaluate.EvaluableExpression
 
 	// Condition is the optional condition between two matcher variables
 	//
@@ -55,6 +61,8 @@ const (
 	StatusMatcher
 	// SizeMatcher matches responses with response size
 	SizeMatcher
+	// DSLMatcher matches based upon dsl syntax
+	DSLMatcher
 )
 
 // MatcherTypes is an table for conversion of matcher type from string.
@@ -64,6 +72,7 @@ var MatcherTypes = map[string]MatcherType{
 	"word":   WordsMatcher,
 	"regex":  RegexMatcher,
 	"binary": BinaryMatcher,
+	"dsl":    DSLMatcher,
 }
 
 // ConditionType is the type of condition for matcher

pkg/matchers/util.go

@@ -0,0 +1,62 @@
+package matchers
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httputil"
+	"strings"
+
+	"github.com/miekg/dns"
+)
+
+func httpToMap(resp *http.Response, body, headers string) (m map[string]interface{}) {
+	m = make(map[string]interface{})
+
+	m["content_length"] = resp.ContentLength
+	m["status_code"] = resp.StatusCode
+	for k, v := range resp.Header {
+		k = strings.ToLower(strings.TrimSpace(strings.Replace(k, "-", "_", -1)))
+		m[k] = strings.Join(v, " ")
+	}
+	m["all_headers"] = headers
+
+	m["body"] = body
+	if r, err := httputil.DumpResponse(resp, true); err == nil {
+		m["raw"] = string(r)
+	}
+
+	return m
+}
+
+func dnsToMap(msg *dns.Msg) (m map[string]interface{}) {
+	m = make(map[string]interface{})
+
+	m["rcode"] = msg.Rcode
+	var qs string
+	for _, question := range msg.Question {
+		qs += fmt.Sprintln(question.String())
+	}
+	m["question"] = qs
+
+	var exs string
+	for _, extra := range msg.Extra {
+		exs += fmt.Sprintln(extra.String())
+	}
+	m["extra"] = exs
+
+	var ans string
+	for _, answer := range msg.Answer {
+		ans += fmt.Sprintln(answer.String())
+	}
+	m["answer"] = ans
+
+	var nss string
+	for _, ns := range msg.Ns {
+		nss += fmt.Sprintln(ns.String())
+	}
+	m["ns"] = nss
+
+	m["raw"] = msg.String()
+
+	return m
+}