daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Finished executer for per-request execution

dev
Ice3man543 2020-04-26 06:33:59 +05:30
parent 4be566192f
commit 5af4c9b2cf
10 changed files with 200 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -5,6 +5,7 @@ go 1.14
require (
github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496
github.com/miekg/dns v1.1.29
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/gologger v1.0.0
github.com/projectdiscovery/retryabledns v1.0.4
github.com/projectdiscovery/retryablehttp-go v1.0.1

2
go.sum
View File

@ -6,6 +6,8 @@ github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRn
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s=

2
internal/runner/runner.go
View File

@ -162,7 +162,7 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
}
// sendRequest sends a request to the target based on a template
func (r *Runner) sendRequest(template *templates.Template, request interface{}, URL string, writer *bufio.Writer, httpclient *retryablehttp.Client, dnsclient *retryabledns.Client) {
func (r *Runner) sendRequest(template *templates.Template, request interface{}, URL string, httpclient *retryablehttp.Client, dnsclient *retryabledns.Client) {
switch request.(type) {
case *requests.HTTPRequest:

97
pkg/executor/executer_http.go
View File

@ -1,13 +1,15 @@
package executor
import (
"bufio"
"crypto/tls"
"io"
"io/ioutil"
"net/http"
"sync"
"time"
"github.com/projectdiscovery/gologger"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/pkg/extractors"
"github.com/projectdiscovery/nuclei/pkg/matchers"
"github.com/projectdiscovery/nuclei/pkg/requests"
@ -21,14 +23,27 @@ type HTTPExecutor struct {
httpClient *retryablehttp.Client
template *templates.Template
httpRequest *requests.HTTPRequest
writer *bufio.Writer
outputMutex *sync.Mutex
}
// HTTPOptions contains configuration options for the HTTP executor.
type HTTPOptions struct {
Template *templates.Template
HTTPRequest *requests.HTTPRequest
Writer *bufio.Writer
Timeout int
Retries int
}
// NewHTTPExecutor creates a new HTTP executor from a template
// and a HTTP request query.
func NewHTTPExecutor(template *templates.Template, httpRequest *requests.HTTPRequest) *HTTPExecutor {
func NewHTTPExecutor(options *HTTPOptions) *HTTPExecutor {
retryablehttpOptions := retryablehttp.DefaultOptionsSpraying
retryablehttpOptions.RetryWaitMax = 10 * time.Second
retryablehttpOptions.RetryMax = r.options.Retries
retryablehttpOptions.RetryMax = options.Retries
followRedirects := options.HTTPRequest.Redirects
maxRedirects := options.HTTPRequest.MaxRedirects
// Create the HTTP Client
client := retryablehttp.NewWithHTTPClient(&http.Client{
@ -40,18 +55,18 @@ func NewHTTPExecutor(template *templates.Template, httpRequest *requests.HTTPReq
},
DisableKeepAlives: true,
},
Timeout: time.Duration(r.options.Timeout) * time.Second,
Timeout: time.Duration(options.Timeout) * time.Second,
CheckRedirect: func(_ *http.Request, requests []*http.Request) error {
if !httpRequest.Redirects {
if !followRedirects {
return http.ErrUseLastResponse
}
if httpRequest.MaxRedirects == 0 {
if maxRedirects == 0 {
if len(requests) > 10 {
return http.ErrUseLastResponse
}
return nil
}
if len(requests) > httpRequest.MaxRedirects {
if len(requests) > maxRedirects {
return http.ErrUseLastResponse
}
return nil
@ -61,48 +76,46 @@ func NewHTTPExecutor(template *templates.Template, httpRequest *requests.HTTPReq
executer := &HTTPExecutor{
httpClient: client,
template: template,
httpRequest: httpRequest,
template: options.Template,
httpRequest: options.HTTPRequest,
outputMutex: &sync.Mutex{},
writer: options.Writer,
}
return executer
}
// ExecuteHTTP executes the HTTP request on a URL
func (e *HTTPExecutor) ExecuteHTTP(URL string) {
if !isURL(URL) {
return
}
func (e *HTTPExecutor) ExecuteHTTP(URL string) error {
// Compile each request for the template based on the URL
compiledRequest, err := e.httpRequest.MakeHTTPRequest(URL)
if err != nil {
gologger.Warningf("[%s] Could not make request %s: %s\n", e.template.ID, URL, err)
return
return errors.Wrap(err, "could not make http request")
}
// Send the request to the target servers
mainLoop:
for _, req := range compiledRequest {
resp, err := e.httpClient.Do(req)
if err != nil {
if resp != nil {
resp.Body.Close()
}
gologger.Warningf("[%s] Could not send request %s: %s\n", e.template.ID, URL, err)
return
return errors.Wrap(err, "could not make http request")
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
gologger.Warningf("[%s] Could not read body %s: %s\n", e.template.ID, URL, err)
continue
return errors.Wrap(err, "could not read http body")
}
resp.Body.Close()
// Convert response body from []byte to string with zero copy
body := unsafeToString(data)
var headers string
matcherCondition := e.httpRequest.GetMatchersCondition()
for _, matcher := range e.httpRequest.Matchers {
// Only build the headers string if the matcher asks for it
part := matcher.GetPart()
@ -111,17 +124,43 @@ func (e *HTTPExecutor) ExecuteHTTP(URL string) {
}
// Check if the matcher matched
if matcher.Match(resp, body, headers) {
// If there is an extractor, run it.
var extractorResults []string
for _, extractor := range e.httpRequest.Extractors {
part := extractor.GetPart()
if part == extractors.AllPart || part == extractors.HeaderPart && headers == "" {
headers = headersToString(resp.Header)
}
extractorResults = append(extractorResults, extractor.Extract(body, headers)...)
if !matcher.Match(resp, body, headers) {
// If the condition is AND we haven't matched, try next request.
if matcherCondition == matchers.ANDCondition {
goto mainLoop
}
} else {
// If the matcher has matched, and its an OR
// write the first output then move to next matcher.
if matcherCondition == matchers.ORCondition && len(e.httpRequest.Extractors) == 0 {
e.writeOutputHTTP(req, matcher, nil)
}
}
}
// All matchers have successfully completed so now start with the
// next task which is extraction of input from matchers.
var extractorResults []string
for _, extractor := range e.httpRequest.Extractors {
part := extractor.GetPart()
if part == extractors.AllPart || part == extractors.HeaderPart && headers == "" {
headers = headersToString(resp.Header)
}
extractorResults = append(extractorResults, extractor.Extract(body, headers)...)
}
// Write a final string of output if matcher type is
// AND or if we have extractors for the mechanism too.
if len(e.httpRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
e.writeOutputHTTP(req, nil, extractorResults)
}
}
return nil
}
// Close closes the http executor for a template.
func (e *HTTPExecutor) Close() {
e.outputMutex.Lock()
e.writer.Flush()
e.outputMutex.Unlock()
}

69
pkg/executor/executor_dns.go
View File

@ -1,7 +1,11 @@
package executor
import (
"github.com/projectdiscovery/gologger"
"bufio"
"sync"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/pkg/matchers"
"github.com/projectdiscovery/nuclei/pkg/requests"
"github.com/projectdiscovery/nuclei/pkg/templates"
retryabledns "github.com/projectdiscovery/retryabledns"
@ -10,9 +14,11 @@ import (
// DNSExecutor is a client for performing a DNS request
// for a template.
type DNSExecutor struct {
dnsClient *retryabledns.Client
template *templates.Template
dnsRequest *requests.DNSRequest
dnsClient *retryabledns.Client
template *templates.Template
dnsRequest *requests.DNSRequest
writer *bufio.Writer
outputMutex *sync.Mutex
}
// DefaultResolvers contains the list of resolvers known to be trusted.
@ -23,21 +29,29 @@ var DefaultResolvers = []string{
"8.8.4.4:53", // Google
}
// DNSOptions contains configuration options for the DNS executor.
type DNSOptions struct {
Template *templates.Template
DNSRequest *requests.DNSRequest
Writer *bufio.Writer
}
// NewDNSExecutor creates a new DNS executor from a template
// and a DNS request query.
func NewDNSExecutor(template *templates.Template, dnsRequest *requests.DNSRequest) *DNSExecutor {
dnsClient := retryabledns.New(DefaultResolvers, dnsRequest.Retries)
func NewDNSExecutor(options *DNSOptions) *DNSExecutor {
dnsClient := retryabledns.New(DefaultResolvers, options.DNSRequest.Retries)
executer := &DNSExecutor{
dnsClient: dnsClient,
template: template,
dnsRequest: dnsRequest,
template: options.Template,
dnsRequest: options.DNSRequest,
writer: options.Writer,
}
return executer
}
// ExecuteDNS executes the DNS request on a URL
func (e *DNSExecutor) ExecuteDNS(URL string) {
func (e *DNSExecutor) ExecuteDNS(URL string) error {
// Parse the URL and return domain if URL.
var domain string
if isURL(URL) {
@ -47,29 +61,52 @@ func (e *DNSExecutor) ExecuteDNS(URL string) {
}
// Compile each request for the template based on the URL
compiledRequest, err := e.dnsRequest.MakeDNSRequest(URL)
compiledRequest, err := e.dnsRequest.MakeDNSRequest(domain)
if err != nil {
gologger.Warningf("[%s] Could not make request %s: %s\n", e.template.ID, domain, err)
return
return errors.Wrap(err, "could not make dns request")
}
// Send the request to the target servers
resp, err := e.dnsClient.Do(compiledRequest)
if err != nil {
gologger.Warningf("[%s] Could not send request %s: %s\n", e.template.ID, domain, err)
return
return errors.Wrap(err, "could not send dns request")
}
matcherCondition := e.dnsRequest.GetMatchersCondition()
for _, matcher := range e.dnsRequest.Matchers {
// Check if the matcher matched
if !matcher.MatchDNS(resp) {
return
// If the condition is AND we haven't matched, return.
if matcherCondition == matchers.ANDCondition {
return nil
}
} else {
// If the matcher has matched, and its an OR
// write the first output then move to next matcher.
if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 {
e.writeOutputDNS(domain, matcher, nil)
}
}
}
// If there is an extractor, run it.
// All matchers have successfully completed so now start with the
// next task which is extraction of input from matchers.
var extractorResults []string
for _, extractor := range e.dnsRequest.Extractors {
extractorResults = append(extractorResults, extractor.ExtractDNS(resp.String())...)
}
// Write a final string of output if matcher type is
// AND or if we have extractors for the mechanism too.
if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
e.writeOutputDNS(domain, nil, extractorResults)
}
return nil
}
// Close closes the dns executor for a template.
func (e *DNSExecutor) Close() {
e.outputMutex.Lock()
e.writer.Flush()
e.outputMutex.Unlock()
}

21
pkg/executor/output_dns.go
View File

@ -2,13 +2,20 @@ package executor
import (
"strings"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/pkg/matchers"
)
// buildOutput builds an output text for writing results
func (e *DNSExecutor) buildOutputDNS(domain string, extractorResults []string) string {
// writeOutputDNS writes dns output to streams
func (e *DNSExecutor) writeOutputDNS(domain string, matcher *matchers.Matcher, extractorResults []string) {
builder := &strings.Builder{}
builder.WriteRune('[')
builder.WriteString(e.template.ID)
if matcher != nil && len(matcher.Name) > 0 {
builder.WriteString(":")
builder.WriteString(matcher.Name)
}
builder.WriteString("] [dns] ")
builder.WriteString(domain)
@ -26,5 +33,13 @@ func (e *DNSExecutor) buildOutputDNS(domain string, extractorResults []string) s
}
builder.WriteRune('\n')
return builder.String()
// Write output to screen as well as any output file
message := builder.String()
gologger.Silentf("%s", message)
if e.writer != nil {
e.outputMutex.Lock()
e.writer.WriteString(message)
e.outputMutex.Unlock()
}
}

17
pkg/executor/output_http.go
View File

@ -3,17 +3,18 @@ package executor
import (
"strings"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/pkg/matchers"
"github.com/projectdiscovery/retryablehttp-go"
)
// buildOutputHTTP builds an output text for writing results
func (e *HTTPExecutor) buildOutputHTTP(req *retryablehttp.Request, extractorResults []string, matcher *matchers.Matcher) string {
// writeOutputHTTP writes http output to streams
func (e *HTTPExecutor) writeOutputHTTP(req *retryablehttp.Request, matcher *matchers.Matcher, extractorResults []string) {
builder := &strings.Builder{}
builder.WriteRune('[')
builder.WriteString(e.template.ID)
if len(matcher.Name) > 0 {
if matcher != nil && len(matcher.Name) > 0 {
builder.WriteString(":")
builder.WriteString(matcher.Name)
}
@ -37,5 +38,13 @@ func (e *HTTPExecutor) buildOutputHTTP(req *retryablehttp.Request, extractorResu
}
builder.WriteRune('\n')
return builder.String()
// Write output to screen as well as any output file
message := builder.String()
gologger.Silentf("%s", message)
if e.writer != nil {
e.outputMutex.Lock()
e.writer.WriteString(message)
e.outputMutex.Unlock()
}
}

15
pkg/requests/dns-request.go
View File

@ -21,11 +21,26 @@ type DNSRequest struct {
// Matchers contains the detection mechanism for the request to identify
// whether the request was successful
Matchers []*matchers.Matcher `yaml:"matchers,omitempty"`
// matchersCondition is internal condition for the matchers.
matchersCondition matchers.ConditionType
// MatchersCondition is the condition of the matchers
// whether to use AND or OR. Default is OR.
MatchersCondition string `yaml:"matchers-condition,omitempty"`
// Extractors contains the extraction mechanism for the request to identify
// and extract parts of the response.
Extractors []*extractors.Extractor `yaml:"extractors,omitempty"`
}
// GetMatchersCondition returns the condition for the matcher
func (r *DNSRequest) GetMatchersCondition() matchers.ConditionType {
return r.matchersCondition
}
// SetMatchersCondition sets the condition for the matcher
func (r *DNSRequest) SetMatchersCondition(condition matchers.ConditionType) {
r.matchersCondition = condition
}
// MakeDNSRequest creates a *dns.Request from a request template
func (r *DNSRequest) MakeDNSRequest(domain string) (*dns.Msg, error) {
domain = dns.Fqdn(domain)

12
pkg/requests/http-request.go
View File

@ -28,6 +28,8 @@ type HTTPRequest struct {
// MatchersCondition is the condition of the matchers
// whether to use AND or OR. Default is OR.
MatchersCondition string `yaml:"matchers-condition,omitempty"`
// matchersCondition is internal condition for the matchers.
matchersCondition matchers.ConditionType
// Extractors contains the extraction mechanism for the request to identify
// and extract parts of the response.
Extractors []*extractors.Extractor `yaml:"extractors,omitempty"`
@ -37,6 +39,16 @@ type HTTPRequest struct {
MaxRedirects int `yaml:"max-redirects,omitempty"`
}
// GetMatchersCondition returns the condition for the matcher
func (r *HTTPRequest) GetMatchersCondition() matchers.ConditionType {
return r.matchersCondition
}
// SetMatchersCondition sets the condition for the matcher
func (r *HTTPRequest) SetMatchersCondition(condition matchers.ConditionType) {
r.matchersCondition = condition
}
// MakeHTTPRequest creates a *http.Request from a request template
func (r *HTTPRequest) MakeHTTPRequest(baseURL string) ([]*retryablehttp.Request, error) {
parsed, err := url.Parse(baseURL)

17
pkg/templates/compile.go
View File

@ -3,6 +3,7 @@ package templates
import (
"os"
"github.com/projectdiscovery/nuclei/pkg/matchers"
"gopkg.in/yaml.v2"
)
@ -24,6 +25,14 @@ func ParseTemplate(file string) (*Template, error) {
// Compile the matchers and the extractors for http requests
for _, request := range template.RequestsHTTP {
// Get the condition between the matchers
condition, ok := matchers.ConditionTypes[request.MatchersCondition]
if !ok {
request.SetMatchersCondition(matchers.ANDCondition)
} else {
request.SetMatchersCondition(condition)
}
for _, matcher := range request.Matchers {
if err = matcher.CompileMatchers(); err != nil {
return nil, err
@ -39,6 +48,14 @@ func ParseTemplate(file string) (*Template, error) {
// Compile the matchers and the extractors for dns requests
for _, request := range template.RequestsDNS {
// Get the condition between the matchers
condition, ok := matchers.ConditionTypes[request.MatchersCondition]
if !ok {
request.SetMatchersCondition(matchers.ANDCondition)
} else {
request.SetMatchersCondition(condition)
}
for _, matcher := range request.Matchers {
if err = matcher.CompileMatchers(); err != nil {
return nil, err