daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merging master

dev
Mzack9999 2020-10-23 10:16:17 +02:00
parent 53e4b45479 9b7de41e75
commit 7ff170221e
6 changed files with 123 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
v2/internal/runner/options.go
View File

@ -39,6 +39,7 @@ type Options struct {
ProxyURL string // ProxyURL is the URL for the proxy server
ProxySocksURL string // ProxySocksURL is the URL for the proxy socks server
TemplatesDirectory string // TemplatesDirectory is the directory to use for storing templates
TraceLogFile string // TraceLogFile specifies a file to write with the trace of all requests
Templates multiStringFlag // Signature specifies the template/templates to use
ExcludedTemplates multiStringFlag // Signature specifies the template/templates to exclude
CustomHeaders requests.CustomHeaders // Custom global headers
@ -77,6 +78,7 @@ func ParseOptions() *Options {
flag.Var(&options.CustomHeaders, "H", "Custom Header.")
flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses")
flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)")
flag.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log")
flag.StringVar(&options.TemplatesDirectory, "update-directory", "", "Directory to use for storing nuclei-templates")
flag.BoolVar(&options.JSON, "json", false, "Write json output to files")
flag.BoolVar(&options.JSONRequests, "json-requests", false, "Write requests/responses for matches in JSON output")

4
v2/internal/runner/processor.go
View File

@ -39,6 +39,7 @@ func (r *Runner) processTemplateWithList(p progress.IProgress, template *templat
switch value := request.(type) {
case *requests.DNSRequest:
dnsExecuter = executer.NewDNSExecuter(&executer.DNSOptions{
TraceLog: r.traceLog,
Debug: r.options.Debug,
Template: template,
DNSRequest: value,
@ -52,6 +53,7 @@ func (r *Runner) processTemplateWithList(p progress.IProgress, template *templat
})
case *requests.BulkHTTPRequest:
httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{
TraceLog: r.traceLog,
Debug: r.options.Debug,
Template: template,
BulkHTTPRequest: value,
@ -214,6 +216,7 @@ func (r *Runner) preloadWorkflowTemplates(p progress.IProgress, workflow *workfl
template := &workflows.Template{Progress: p}
if len(t.BulkRequestsHTTP) > 0 {
template.HTTPOptions = &executer.HTTPOptions{
TraceLog: r.traceLog,
Debug: r.options.Debug,
Writer: r.output,
Template: t,
@ -231,6 +234,7 @@ func (r *Runner) preloadWorkflowTemplates(p progress.IProgress, workflow *workfl
}
} else if len(t.RequestsDNS) > 0 {
template.DNSOptions = &executer.DNSOptions{
TraceLog: r.traceLog,
Debug: r.options.Debug,
Template: t,
Writer: r.output,

16
v2/internal/runner/runner.go
View File

@ -11,9 +11,11 @@ import (
"sync"
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
"github.com/projectdiscovery/nuclei/v2/pkg/collaborator"
"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
@ -26,11 +28,13 @@ import (
type Runner struct {
input string
inputCount int64
tempFile string
traceLog tracelog.Log
// output is the output file to write if any
output *bufwriter.Writer
tempFile string
templatesConfig *nucleiConfig
// options contains configuration options for runner
options *Options
@ -46,7 +50,15 @@ type Runner struct {
// New creates a new client for running enumeration process.
func New(options *Options) (*Runner, error) {
runner := &Runner{
options: options,
traceLog: &tracelog.NoopLogger{},
options: options,
}
if options.TraceLogFile != "" {
fileLog, err := tracelog.NewFileLogger(options.TraceLogFile)
if err != nil {
return nil, errors.Wrap(err, "could not create file trace logger")
}
runner.traceLog = fileLog
}
if err := runner.updateTemplates(); err != nil {

76
v2/internal/tracelog/tracelog.go Normal file
View File

@ -0,0 +1,76 @@
package tracelog
import (
"os"
"sync"
jsoniter "github.com/json-iterator/go"
)
// Log is an interface for logging trace log of all the requests
type Log interface {
// Close closes the log interface flushing data
Close()
// Request writes a log the requests trace log
Request(templateID, url, requestType string, err error)
}
// NoopLogger is a noop logger that simply does nothing
type NoopLogger struct{}
// Close closes the log interface flushing data
func (n *NoopLogger) Close() {}
// Request writes a log the requests trace log
func (n *NoopLogger) Request(templateID, url, requestType string, err error) {}
// FileLogger is a trace logger that writes request logs to a file.
type FileLogger struct {
encoder *jsoniter.Encoder
file *os.File
mutex *sync.Mutex
}
// NewFileLogger creates a new file logger structure
func NewFileLogger(path string) (*FileLogger, error) {
file, err := os.Create(path)
if err != nil {
return nil, err
}
return &FileLogger{file: file, encoder: jsoniter.NewEncoder(file), mutex: &sync.Mutex{}}, nil
}
// Close closes the log interface flushing data
func (f *FileLogger) Close() {
f.mutex.Lock()
defer f.mutex.Unlock()
f.file.Close()
}
// JSONRequest is a trace log request written to file
type JSONRequest struct {
ID string `json:"id"`
URL string `json:"url"`
Error string `json:"error"`
Type string `json:"type"`
}
// Request writes a log the requests trace log
func (f *FileLogger) Request(templateID, url, requestType string, err error) {
request := &JSONRequest{
ID: templateID,
URL: url,
Type: requestType,
}
if err != nil {
request.Error = err.Error()
} else {
request.Error = "none"
}
f.mutex.Lock()
defer f.mutex.Unlock()
//nolint:errcheck // We don't need to do anything here
f.encoder.Encode(request)
}

6
v2/pkg/executer/executer_dns.go
View File

@ -9,6 +9,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/requests"
@ -25,6 +26,7 @@ type DNSExecuter struct {
jsonRequest bool
noMeta bool
Results bool
traceLog tracelog.Log
dnsClient *retryabledns.Client
template *templates.Template
dnsRequest *requests.DNSRequest
@ -49,6 +51,7 @@ type DNSOptions struct {
JSON bool
JSONRequests bool
NoMeta bool
TraceLog tracelog.Log
Template *templates.Template
DNSRequest *requests.DNSRequest
Writer *bufwriter.Writer
@ -66,6 +69,7 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
debug: options.Debug,
noMeta: options.NoMeta,
jsonOutput: options.JSON,
traceLog: options.TraceLog,
jsonRequest: options.JSONRequests,
dnsClient: dnsClient,
template: options.Template,
@ -94,12 +98,14 @@ func (e *DNSExecuter) ExecuteDNS(p progress.IProgress, reqURL string) *Result {
// Compile each request for the template based on the URL
compiledRequest, err := e.dnsRequest.MakeDNSRequest(domain)
if err != nil {
e.traceLog.Request(e.template.ID, domain, "dns", err)
result.Error = errors.Wrap(err, "could not make dns request")
p.Drop(1)
return result
}
e.traceLog.Request(e.template.ID, domain, "dns", nil)
if e.debug {
gologger.Infof("Dumped DNS request for %s (%s)\n\n", reqURL, e.template.ID)

23
v2/pkg/executer/executer_http.go
View File

@ -22,6 +22,7 @@ import (
"github.com/projectdiscovery/httpx/common/cache"
"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
"github.com/projectdiscovery/nuclei/v2/pkg/globalratelimiter"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
@ -49,6 +50,7 @@ type HTTPExecuter struct {
bulkHTTPRequest *requests.BulkHTTPRequest
writer *bufwriter.Writer
CookieJar *cookiejar.Jar
traceLog tracelog.Log
decolorizer *regexp.Regexp
coloredOutput bool
debug bool
@ -72,6 +74,7 @@ type HTTPOptions struct {
CookieJar *cookiejar.Jar
Colorizer *colorizer.NucleiColorizer
Decolorizer *regexp.Regexp
TraceLog tracelog.Log
Debug bool
JSON bool
JSONRequests bool
@ -124,6 +127,7 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
noMeta: options.NoMeta,
httpClient: client,
rawHTTPClient: rawClient,
traceLog: options.TraceLog,
template: options.Template,
bulkHTTPRequest: options.BulkHTTPRequest,
writer: options.Writer,
@ -170,10 +174,13 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p progress.IProgress, reqURL string)
globalratelimiter.Take(reqURL)
// If the request was built correctly then execute it
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result)
err := e.handleHTTP(reqURL, httpRequest, dynamicvalues, result)
if err != nil {
e.traceLog.Request(e.template.ID, reqURL, "http", err)
result.Error = errors.Wrap(err, "could not handle http request")
p.Drop(remaining)
} else {
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
}
}(request)
}
@ -241,8 +248,11 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(p progress.IProgress, reqURL string) *Re
request.PipelineClient = pipeclient
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result)
if err != nil {
e.traceLog.Request(e.template.ID, reqURL, "http", err)
result.Error = errors.Wrap(err, "could not handle http request")
p.Drop(remaining)
} else {
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
}
request.PipelineClient = nil
}(request)
@ -291,10 +301,13 @@ func (e *HTTPExecuter) ExecuteHTTP(p progress.IProgress, reqURL string) *Result
} else {
globalratelimiter.Take(reqURL)
// If the request was built correctly then execute it
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result)
err := e.handleHTTP(reqURL, httpRequest, dynamicvalues, result)
if err != nil {
result.Error = errors.Wrap(err, "could not handle http request")
p.Drop(remaining)
e.traceLog.Request(e.template.ID, reqURL, "http", err)
} else {
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
}
}
@ -340,8 +353,10 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
if resp != nil {
resp.Body.Close()
}
e.traceLog.Request(e.template.ID, reqURL, "http", err)
return err
}
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
} else if request.Unsafe {
// rawhttp
// burp uses "\r\n" as new line character
@ -355,8 +370,10 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
if resp != nil {
resp.Body.Close()
}
e.traceLog.Request(e.template.ID, reqURL, "http", err)
return err
}
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
} else {
// retryablehttp
resp, err = e.httpClient.Do(request.Request)
@ -364,8 +381,10 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
if resp != nil {
resp.Body.Close()
}
e.traceLog.Request(e.template.ID, reqURL, "http", err)
return err
}
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
}
duration := time.Since(timeStart)