Added extractors to the workflow + misc

go.mod

@@ -5,6 +5,7 @@ go 1.14
 require (
 	github.com/projectdiscovery/gologger v1.0.0
 	github.com/projectdiscovery/retryablehttp-go v0.0.0-20200404173421-61821d9501d8
+	github.com/stretchr/testify v1.5.1
 	github.com/valyala/fasttemplate v1.1.0
 	gopkg.in/yaml.v2 v2.2.8
 )

go.sum

@@ -1,17 +1,21 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s=
 github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
 github.com/projectdiscovery/retryablehttp-go v0.0.0-20200404173421-61821d9501d8 h1:7JsD0gLcUalU147Y4Vq6OGQp/zyFIyjE63oeYa1QPpQ=
 github.com/projectdiscovery/retryablehttp-go v0.0.0-20200404173421-61821d9501d8/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.1.0 h1:RZqt0yGBsps8NGvLSGW804QQqCUYYLsaOjTVHy1Ocw4=
 github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=

internal/runner/runner.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/pkg/extractors"
 	"github.com/projectdiscovery/nuclei/pkg/matchers"
 	"github.com/projectdiscovery/nuclei/pkg/templates"
 	retryablehttp "github.com/projectdiscovery/retryablehttp-go"
@@ -198,12 +199,10 @@ func (r *Runner) sendRequest(template *templates.Template, URL string, writer *b
 			var headers string
 			for _, matcher := range request.Matchers {
 				// Only build the headers string if the matcher asks for it
-				switch matcher.GetPart() {
-				case matchers.AllPart, matchers.HeaderPart:
-					if headers == "" {
+				part := matcher.GetPart()
+				if part == matchers.AllPart || part == matchers.HeaderPart && headers == "" {
 					headers = headersToString(resp.Header)
 				}
-				}
 
 				// Check if the matcher matched
 				if !matcher.Match(resp, body, headers) {
@@ -211,8 +210,18 @@ func (r *Runner) sendRequest(template *templates.Template, URL string, writer *b
 				}
 			}
 
+			// If there is an extractor, run it.
+			var extractorResults []string
+			for _, extractor := range request.Extractors {
+				part := extractor.GetPart()
+				if part == extractors.AllPart || part == extractors.HeaderPart && headers == "" {
+					headers = headersToString(resp.Header)
+				}
+				extractorResults = append(extractorResults, extractor.Extract(body, headers)...)
+			}
+
 			// All the matchers matched, print the output on the screen
-			output := fmt.Sprintf("[%s] %s\n", template.ID, req.URL.String())
+			output := buildOutput(template, req, extractorResults)
 			gologger.Silentf("%s", output)
 
 			if writer != nil {
@@ -223,3 +232,27 @@ func (r *Runner) sendRequest(template *templates.Template, URL string, writer *b
 		}
 	}
 }
+
+// buildOutput builds an output text for writing results
+func buildOutput(template *templates.Template, req *retryablehttp.Request, extractorResults []string) string {
+	builder := &strings.Builder{}
+	builder.WriteRune('[')
+	builder.WriteString(template.ID)
+	builder.WriteString("] ")
+	builder.WriteString(req.URL.String())
+
+	// If any extractors, write the results
+	if len(extractorResults) > 0 {
+		builder.WriteString(" [")
+		for i, result := range extractorResults {
+			builder.WriteString(result)
+			if i != len(extractorResults)-1 {
+				builder.WriteRune(',')
+			}
+		}
+		builder.WriteString("]")
+	}
+	builder.WriteRune('\n')
+
+	return builder.String()
+}

pkg/extractors/compile.go

@@ -0,0 +1,35 @@
+package extractors
+
+import (
+	"fmt"
+	"regexp"
+)
+
+// CompileExtractors performs the initial setup operation on a extractor
+func (e *Extractor) CompileExtractors() error {
+	// Setup the matcher type
+	_, ok := ExtractorTypes[e.Type]
+	if !ok {
+		return fmt.Errorf("unknown extractor type specified: %s", e.Type)
+	}
+
+	// Compile the regexes
+	for _, regex := range e.Regex {
+		compiled, err := regexp.Compile(regex)
+		if err != nil {
+			return fmt.Errorf("could not compile regex: %s", regex)
+		}
+		e.regexCompiled = append(e.regexCompiled, compiled)
+	}
+
+	// Setup the part of the request to match, if any.
+	if e.Part != "" {
+		e.part, ok = PartTypes[e.Part]
+		if !ok {
+			return fmt.Errorf("unknown matcher part specified: %s", e.Part)
+		}
+	} else {
+		e.part = BodyPart
+	}
+	return nil
+}

pkg/extractors/extract.go

@@ -18,5 +18,9 @@ func (e *Extractor) Extract(body, headers string) []string {
 
 // extractRegex extracts text from a corpus and returns it
 func (e *Extractor) extractRegex(corpus string) []string {
-	return e.regexCompiled.FindAllString(corpus, -1)
+	results := []string{}
+	for _, regex := range e.regexCompiled {
+		results = append(results, regex.FindAllString(corpus, -1)...)
+	}
+	return results
 }

pkg/extractors/extractors.go

@@ -4,10 +4,13 @@ import "regexp"
 
 // Extractor is used to extract part of response using a regex.
 type Extractor struct {
+	// Type is the type of the matcher
+	Type string `yaml:"type"`
+
 	// Regex are the regex pattern required to be present in the response
-	Regex string `yaml:"regex"`
+	Regex []string `yaml:"regex"`
 	// regexCompiled is the compiled variant
-	regexCompiled *regexp.Regexp
+	regexCompiled []*regexp.Regexp
 
 	// Part is the part of the request to match
 	//
@@ -17,6 +20,19 @@ type Extractor struct {
 	part Part
 }
 
+// ExtractorType is the type of the extractor specified
+type ExtractorType = int
+
+const (
+	// RegexExtractor extracts responses with regexes
+	RegexExtractor ExtractorType = iota + 1
+)
+
+// ExtractorTypes is an table for conversion of extractor type from string.
+var ExtractorTypes = map[string]ExtractorType{
+	"regex": RegexExtractor,
+}
+
 // Part is the part of the request to match
 type Part int
 

pkg/requests/requests.go

@@ -6,6 +6,7 @@ import (
 	"net/url"
 	"strings"
 
+	"github.com/projectdiscovery/nuclei/pkg/extractors"
 	"github.com/projectdiscovery/nuclei/pkg/matchers"
 	retryablehttp "github.com/projectdiscovery/retryablehttp-go"
 	"github.com/valyala/fasttemplate"
@@ -26,7 +27,7 @@ type Request struct {
 	Matchers []*matchers.Matcher `yaml:"matchers,omitempty"`
 	// Extractors contains the extraction mechanism for the request to identify
 	// and extract parts of the response.
-	Extractors []*matchers.Matcher `yaml:"extractors,omitempty"`
+	Extractors []*extractors.Extractor `yaml:"extractors,omitempty"`
 }
 
 // MakeRequest creates a *http.Request from a request template

pkg/templates/compile.go

@@ -22,12 +22,19 @@ func ParseTemplate(file string) (*Template, error) {
 	}
 	f.Close()
 
+	// Compile the matchers and the extractors
 	for _, request := range template.Requests {
 		for _, matcher := range request.Matchers {
 			if err = matcher.CompileMatchers(); err != nil {
 				return nil, err
 			}
 		}
+
+		for _, extractor := range request.Extractors {
+			if err := extractor.CompileExtractors(); err != nil {
+				return nil, err
+			}
+		}
 	}
 	return template, nil
 }