From f5435e6c9a3501f000f72c300249e14c7b318838 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 22 Oct 2020 16:12:16 +0530 Subject: [PATCH 1/3] Added trace log feature to write execution log --- v2/internal/runner/options.go | 2 + v2/internal/runner/processor.go | 4 ++ v2/internal/runner/runner.go | 16 ++++++- v2/internal/tracelog/tracelog.go | 73 ++++++++++++++++++++++++++++++++ v2/pkg/executer/executer_dns.go | 6 +++ v2/pkg/executer/executer_http.go | 23 +++++++++- 6 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 v2/internal/tracelog/tracelog.go diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 32d74b2e..1c94b755 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -39,6 +39,7 @@ type Options struct { ProxyURL string // ProxyURL is the URL for the proxy server ProxySocksURL string // ProxySocksURL is the URL for the proxy socks server TemplatesDirectory string // TemplatesDirectory is the directory to use for storing templates + TraceLogFile string // TraceLogFile specifies a file to write with the trace of all requests Templates multiStringFlag // Signature specifies the template/templates to use ExcludedTemplates multiStringFlag // Signature specifies the template/templates to exclude CustomHeaders requests.CustomHeaders // Custom global headers @@ -76,6 +77,7 @@ func ParseOptions() *Options { flag.Var(&options.CustomHeaders, "H", "Custom Header.") flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses") flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)") + flag.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log") flag.StringVar(&options.TemplatesDirectory, "update-directory", "", "Directory to use for storing nuclei-templates") flag.BoolVar(&options.JSON, "json", false, "Write json output to files") flag.BoolVar(&options.JSONRequests, "json-requests", false, "Write requests/responses for matches in JSON output") diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 990f5d64..27d9df9e 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -39,6 +39,7 @@ func (r *Runner) processTemplateWithList(p progress.IProgress, template *templat switch value := request.(type) { case *requests.DNSRequest: dnsExecuter = executer.NewDNSExecuter(&executer.DNSOptions{ + TraceLog: r.traceLog, Debug: r.options.Debug, Template: template, DNSRequest: value, @@ -52,6 +53,7 @@ func (r *Runner) processTemplateWithList(p progress.IProgress, template *templat }) case *requests.BulkHTTPRequest: httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{ + TraceLog: r.traceLog, Debug: r.options.Debug, Template: template, BulkHTTPRequest: value, @@ -214,6 +216,7 @@ func (r *Runner) preloadWorkflowTemplates(p progress.IProgress, workflow *workfl template := &workflows.Template{Progress: p} if len(t.BulkRequestsHTTP) > 0 { template.HTTPOptions = &executer.HTTPOptions{ + TraceLog: r.traceLog, Debug: r.options.Debug, Writer: r.output, Template: t, @@ -231,6 +234,7 @@ func (r *Runner) preloadWorkflowTemplates(p progress.IProgress, workflow *workfl } } else if len(t.RequestsDNS) > 0 { template.DNSOptions = &executer.DNSOptions{ + TraceLog: r.traceLog, Debug: r.options.Debug, Template: t, Writer: r.output, diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index f45beddd..93485d9d 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -11,9 +11,11 @@ import ( "sync" "github.com/logrusorgru/aurora" + "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/bufwriter" "github.com/projectdiscovery/nuclei/v2/internal/progress" + "github.com/projectdiscovery/nuclei/v2/internal/tracelog" "github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean" "github.com/projectdiscovery/nuclei/v2/pkg/colorizer" "github.com/projectdiscovery/nuclei/v2/pkg/globalratelimiter" @@ -25,11 +27,13 @@ import ( type Runner struct { input string inputCount int64 + tempFile string + + traceLog tracelog.Log // output is the output file to write if any output *bufwriter.Writer - tempFile string templatesConfig *nucleiConfig // options contains configuration options for runner options *Options @@ -45,7 +49,15 @@ type Runner struct { // New creates a new client for running enumeration process. func New(options *Options) (*Runner, error) { runner := &Runner{ - options: options, + traceLog: &tracelog.NoopLogger{}, + options: options, + } + if options.TraceLogFile != "" { + fileLog, err := tracelog.NewFileLogger(options.TraceLogFile) + if err != nil { + return nil, errors.Wrap(err, "could not create file trace logger") + } + runner.traceLog = fileLog } if err := runner.updateTemplates(); err != nil { diff --git a/v2/internal/tracelog/tracelog.go b/v2/internal/tracelog/tracelog.go new file mode 100644 index 00000000..a9348670 --- /dev/null +++ b/v2/internal/tracelog/tracelog.go @@ -0,0 +1,73 @@ +package tracelog + +import ( + "os" + "sync" + + jsoniter "github.com/json-iterator/go" +) + +// Log is an interface for logging trace log of all the requests +type Log interface { + // Close closes the log interface flushing data + Close() + // Request writes a log the requests trace log + Request(templateID, url string, err error) +} + +// NoopLogger is a noop logger that simply does nothing +type NoopLogger struct{} + +// Close closes the log interface flushing data +func (n *NoopLogger) Close() {} + +// Request writes a log the requests trace log +func (n *NoopLogger) Request(templateID, url string, err error) {} + +// FileLogger is a trace logger that writes request logs to a file. +type FileLogger struct { + encoder *jsoniter.Encoder + file *os.File + mutex *sync.Mutex +} + +// NewFileLogger creates a new file logger structure +func NewFileLogger(path string) (*FileLogger, error) { + file, err := os.Create(path) + if err != nil { + return nil, err + } + return &FileLogger{file: file, encoder: jsoniter.NewEncoder(file), mutex: &sync.Mutex{}}, nil +} + +// Close closes the log interface flushing data +func (f *FileLogger) Close() { + f.mutex.Lock() + defer f.mutex.Unlock() + + f.file.Close() +} + +// JSONRequest is a trace log request written to file +type JSONRequest struct { + ID string `json:"id"` + URL string `json:"url"` + Error string `json:"error"` +} + +// Request writes a log the requests trace log +func (f *FileLogger) Request(templateID, url string, err error) { + request := &JSONRequest{ + ID: templateID, + URL: url, + } + if err != nil { + request.Error = err.Error() + } else { + request.Error = "none" + } + + f.mutex.Lock() + defer f.mutex.Unlock() + f.encoder.Encode(request) +} diff --git a/v2/pkg/executer/executer_dns.go b/v2/pkg/executer/executer_dns.go index 23d6ec5d..48016764 100644 --- a/v2/pkg/executer/executer_dns.go +++ b/v2/pkg/executer/executer_dns.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/bufwriter" "github.com/projectdiscovery/nuclei/v2/internal/progress" + "github.com/projectdiscovery/nuclei/v2/internal/tracelog" "github.com/projectdiscovery/nuclei/v2/pkg/colorizer" "github.com/projectdiscovery/nuclei/v2/pkg/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/requests" @@ -25,6 +26,7 @@ type DNSExecuter struct { jsonRequest bool noMeta bool Results bool + traceLog tracelog.Log dnsClient *retryabledns.Client template *templates.Template dnsRequest *requests.DNSRequest @@ -49,6 +51,7 @@ type DNSOptions struct { JSON bool JSONRequests bool NoMeta bool + TraceLog tracelog.Log Template *templates.Template DNSRequest *requests.DNSRequest Writer *bufwriter.Writer @@ -66,6 +69,7 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter { debug: options.Debug, noMeta: options.NoMeta, jsonOutput: options.JSON, + traceLog: options.TraceLog, jsonRequest: options.JSONRequests, dnsClient: dnsClient, template: options.Template, @@ -94,12 +98,14 @@ func (e *DNSExecuter) ExecuteDNS(p progress.IProgress, reqURL string) *Result { // Compile each request for the template based on the URL compiledRequest, err := e.dnsRequest.MakeDNSRequest(domain) if err != nil { + e.traceLog.Request(e.template.ID, domain, err) result.Error = errors.Wrap(err, "could not make dns request") p.Drop(1) return result } + e.traceLog.Request(e.template.ID, domain, nil) if e.debug { gologger.Infof("Dumped DNS request for %s (%s)\n\n", reqURL, e.template.ID) diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index ea023aec..8bb5922d 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -22,6 +22,7 @@ import ( "github.com/projectdiscovery/httpx/common/cache" "github.com/projectdiscovery/nuclei/v2/internal/bufwriter" "github.com/projectdiscovery/nuclei/v2/internal/progress" + "github.com/projectdiscovery/nuclei/v2/internal/tracelog" "github.com/projectdiscovery/nuclei/v2/pkg/colorizer" "github.com/projectdiscovery/nuclei/v2/pkg/globalratelimiter" "github.com/projectdiscovery/nuclei/v2/pkg/matchers" @@ -49,6 +50,7 @@ type HTTPExecuter struct { bulkHTTPRequest *requests.BulkHTTPRequest writer *bufwriter.Writer CookieJar *cookiejar.Jar + traceLog tracelog.Log decolorizer *regexp.Regexp coloredOutput bool debug bool @@ -72,6 +74,7 @@ type HTTPOptions struct { CookieJar *cookiejar.Jar Colorizer *colorizer.NucleiColorizer Decolorizer *regexp.Regexp + TraceLog tracelog.Log Debug bool JSON bool JSONRequests bool @@ -124,6 +127,7 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) { noMeta: options.NoMeta, httpClient: client, rawHTTPClient: rawClient, + traceLog: options.TraceLog, template: options.Template, bulkHTTPRequest: options.BulkHTTPRequest, writer: options.Writer, @@ -170,10 +174,13 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p progress.IProgress, reqURL string) globalratelimiter.Take(reqURL) // If the request was built correctly then execute it - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result) + err := e.handleHTTP(reqURL, httpRequest, dynamicvalues, result) if err != nil { + e.traceLog.Request(e.template.ID, reqURL, err) result.Error = errors.Wrap(err, "could not handle http request") p.Drop(remaining) + } else { + e.traceLog.Request(e.template.ID, reqURL, nil) } }(request) } @@ -241,8 +248,11 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(p progress.IProgress, reqURL string) *Re request.PipelineClient = pipeclient err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result) if err != nil { + e.traceLog.Request(e.template.ID, reqURL, err) result.Error = errors.Wrap(err, "could not handle http request") p.Drop(remaining) + } else { + e.traceLog.Request(e.template.ID, reqURL, nil) } request.PipelineClient = nil }(request) @@ -291,10 +301,13 @@ func (e *HTTPExecuter) ExecuteHTTP(p progress.IProgress, reqURL string) *Result } else { globalratelimiter.Take(reqURL) // If the request was built correctly then execute it - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result) + err := e.handleHTTP(reqURL, httpRequest, dynamicvalues, result) if err != nil { result.Error = errors.Wrap(err, "could not handle http request") p.Drop(remaining) + e.traceLog.Request(e.template.ID, reqURL, err) + } else { + e.traceLog.Request(e.template.ID, reqURL, nil) } } @@ -340,8 +353,10 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, if resp != nil { resp.Body.Close() } + e.traceLog.Request(e.template.ID, reqURL, err) return err } + e.traceLog.Request(e.template.ID, reqURL, nil) } else if request.Unsafe { // rawhttp // burp uses "\r\n" as new line character @@ -355,8 +370,10 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, if resp != nil { resp.Body.Close() } + e.traceLog.Request(e.template.ID, reqURL, err) return err } + e.traceLog.Request(e.template.ID, reqURL, nil) } else { // retryablehttp resp, err = e.httpClient.Do(request.Request) @@ -364,8 +381,10 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, if resp != nil { resp.Body.Close() } + e.traceLog.Request(e.template.ID, reqURL, err) return err } + e.traceLog.Request(e.template.ID, reqURL, nil) } duration := time.Since(timeStart) From 4e48a5f147e5012f291e3045c7c86d301fcdde8f Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 22 Oct 2020 16:16:33 +0530 Subject: [PATCH 2/3] Added type to specify type of request --- v2/internal/tracelog/tracelog.go | 12 +++++++----- v2/pkg/executer/executer_dns.go | 4 ++-- v2/pkg/executer/executer_http.go | 24 ++++++++++++------------ 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/v2/internal/tracelog/tracelog.go b/v2/internal/tracelog/tracelog.go index a9348670..7d81c42a 100644 --- a/v2/internal/tracelog/tracelog.go +++ b/v2/internal/tracelog/tracelog.go @@ -12,7 +12,7 @@ type Log interface { // Close closes the log interface flushing data Close() // Request writes a log the requests trace log - Request(templateID, url string, err error) + Request(templateID, url, Type string, err error) } // NoopLogger is a noop logger that simply does nothing @@ -22,7 +22,7 @@ type NoopLogger struct{} func (n *NoopLogger) Close() {} // Request writes a log the requests trace log -func (n *NoopLogger) Request(templateID, url string, err error) {} +func (n *NoopLogger) Request(templateID, url, Type string, err error) {} // FileLogger is a trace logger that writes request logs to a file. type FileLogger struct { @@ -53,13 +53,15 @@ type JSONRequest struct { ID string `json:"id"` URL string `json:"url"` Error string `json:"error"` + Type string `json:"type"` } // Request writes a log the requests trace log -func (f *FileLogger) Request(templateID, url string, err error) { +func (f *FileLogger) Request(templateID, url, Type string, err error) { request := &JSONRequest{ - ID: templateID, - URL: url, + ID: templateID, + URL: url, + Type: Type, } if err != nil { request.Error = err.Error() diff --git a/v2/pkg/executer/executer_dns.go b/v2/pkg/executer/executer_dns.go index 48016764..e15dff0b 100644 --- a/v2/pkg/executer/executer_dns.go +++ b/v2/pkg/executer/executer_dns.go @@ -98,14 +98,14 @@ func (e *DNSExecuter) ExecuteDNS(p progress.IProgress, reqURL string) *Result { // Compile each request for the template based on the URL compiledRequest, err := e.dnsRequest.MakeDNSRequest(domain) if err != nil { - e.traceLog.Request(e.template.ID, domain, err) + e.traceLog.Request(e.template.ID, domain, "dns", err) result.Error = errors.Wrap(err, "could not make dns request") p.Drop(1) return result } - e.traceLog.Request(e.template.ID, domain, nil) + e.traceLog.Request(e.template.ID, domain, "dns", nil) if e.debug { gologger.Infof("Dumped DNS request for %s (%s)\n\n", reqURL, e.template.ID) diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 8bb5922d..a6f61297 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -176,11 +176,11 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p progress.IProgress, reqURL string) // If the request was built correctly then execute it err := e.handleHTTP(reqURL, httpRequest, dynamicvalues, result) if err != nil { - e.traceLog.Request(e.template.ID, reqURL, err) + e.traceLog.Request(e.template.ID, reqURL, "http", err) result.Error = errors.Wrap(err, "could not handle http request") p.Drop(remaining) } else { - e.traceLog.Request(e.template.ID, reqURL, nil) + e.traceLog.Request(e.template.ID, reqURL, "http", nil) } }(request) } @@ -248,11 +248,11 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(p progress.IProgress, reqURL string) *Re request.PipelineClient = pipeclient err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result) if err != nil { - e.traceLog.Request(e.template.ID, reqURL, err) + e.traceLog.Request(e.template.ID, reqURL, "http", err) result.Error = errors.Wrap(err, "could not handle http request") p.Drop(remaining) } else { - e.traceLog.Request(e.template.ID, reqURL, nil) + e.traceLog.Request(e.template.ID, reqURL, "http", nil) } request.PipelineClient = nil }(request) @@ -305,9 +305,9 @@ func (e *HTTPExecuter) ExecuteHTTP(p progress.IProgress, reqURL string) *Result if err != nil { result.Error = errors.Wrap(err, "could not handle http request") p.Drop(remaining) - e.traceLog.Request(e.template.ID, reqURL, err) + e.traceLog.Request(e.template.ID, reqURL, "http", err) } else { - e.traceLog.Request(e.template.ID, reqURL, nil) + e.traceLog.Request(e.template.ID, reqURL, "http", nil) } } @@ -353,10 +353,10 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, if resp != nil { resp.Body.Close() } - e.traceLog.Request(e.template.ID, reqURL, err) + e.traceLog.Request(e.template.ID, reqURL, "http", err) return err } - e.traceLog.Request(e.template.ID, reqURL, nil) + e.traceLog.Request(e.template.ID, reqURL, "http", nil) } else if request.Unsafe { // rawhttp // burp uses "\r\n" as new line character @@ -370,10 +370,10 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, if resp != nil { resp.Body.Close() } - e.traceLog.Request(e.template.ID, reqURL, err) + e.traceLog.Request(e.template.ID, reqURL, "http", err) return err } - e.traceLog.Request(e.template.ID, reqURL, nil) + e.traceLog.Request(e.template.ID, reqURL, "http", nil) } else { // retryablehttp resp, err = e.httpClient.Do(request.Request) @@ -381,10 +381,10 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, if resp != nil { resp.Body.Close() } - e.traceLog.Request(e.template.ID, reqURL, err) + e.traceLog.Request(e.template.ID, reqURL, "http", err) return err } - e.traceLog.Request(e.template.ID, reqURL, nil) + e.traceLog.Request(e.template.ID, reqURL, "http", nil) } duration := time.Since(timeStart) From 4aa2d493ca585d403ada42979a1b297100c4b1ab Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 22 Oct 2020 16:20:34 +0530 Subject: [PATCH 3/3] Misc --- v2/internal/tracelog/tracelog.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/v2/internal/tracelog/tracelog.go b/v2/internal/tracelog/tracelog.go index 7d81c42a..ef821581 100644 --- a/v2/internal/tracelog/tracelog.go +++ b/v2/internal/tracelog/tracelog.go @@ -12,7 +12,7 @@ type Log interface { // Close closes the log interface flushing data Close() // Request writes a log the requests trace log - Request(templateID, url, Type string, err error) + Request(templateID, url, requestType string, err error) } // NoopLogger is a noop logger that simply does nothing @@ -22,7 +22,7 @@ type NoopLogger struct{} func (n *NoopLogger) Close() {} // Request writes a log the requests trace log -func (n *NoopLogger) Request(templateID, url, Type string, err error) {} +func (n *NoopLogger) Request(templateID, url, requestType string, err error) {} // FileLogger is a trace logger that writes request logs to a file. type FileLogger struct { @@ -57,11 +57,11 @@ type JSONRequest struct { } // Request writes a log the requests trace log -func (f *FileLogger) Request(templateID, url, Type string, err error) { +func (f *FileLogger) Request(templateID, url, requestType string, err error) { request := &JSONRequest{ ID: templateID, URL: url, - Type: Type, + Type: requestType, } if err != nil { request.Error = err.Error() @@ -71,5 +71,6 @@ func (f *FileLogger) Request(templateID, url, Type string, err error) { f.mutex.Lock() defer f.mutex.Unlock() + //nolint:errcheck // We don't need to do anything here f.encoder.Encode(request) }