Merge pull request #107 from projectdiscovery/iceman-add-debug-verbose

Added better debug and verbose modes

internal/runner/options.go

@@ -11,6 +11,7 @@ import (
 // Options contains the configuration options for tuning
 // the template requesting process.
 type Options struct {
+	Debug         bool                   // Debug mode allows debugging request/responses for the engine
 	Templates     string                 // Signature specifies the template/templates to use
 	Targets       string                 // Targets specifies the targets to scan using templates.
 	Threads       int                    // Thread controls the number of concurrent requests to make.
@@ -45,6 +46,7 @@ func ParseOptions() *Options {
 	flag.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout")
 	flag.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
 	flag.Var(&options.CustomHeaders, "H", "Custom Header.")
+	flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses")
 
 	flag.Parse()
 

internal/runner/runner.go

@@ -189,12 +189,14 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
 	switch value := request.(type) {
 	case *requests.DNSRequest:
 		dnsExecutor = executor.NewDNSExecutor(&executor.DNSOptions{
+			Debug:      r.options.Debug,
 			Template:   template,
 			DNSRequest: value,
 			Writer:     writer,
 		})
 	case *requests.HTTPRequest:
 		httpExecutor, err = executor.NewHTTPExecutor(&executor.HTTPOptions{
+			Debug:         r.options.Debug,
 			Template:      template,
 			HTTPRequest:   value,
 			Writer:        writer,

pkg/executor/executer_http.go

@@ -7,12 +7,15 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
+	"os"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/pkg/extractors"
 	"github.com/projectdiscovery/nuclei/pkg/matchers"
 	"github.com/projectdiscovery/nuclei/pkg/requests"
@@ -24,6 +27,7 @@ import (
 // HTTPExecutor is client for performing HTTP requests
 // for a template.
 type HTTPExecutor struct {
+	debug         bool
 	httpClient    *retryablehttp.Client
 	template      *templates.Template
 	httpRequest   *requests.HTTPRequest
@@ -41,6 +45,7 @@ type HTTPOptions struct {
 	Retries       int
 	ProxyURL      string
 	ProxySocksURL string
+	Debug         bool
 	CustomHeaders requests.CustomHeaders
 }
 
@@ -62,6 +67,7 @@ func NewHTTPExecutor(options *HTTPOptions) (*HTTPExecutor, error) {
 	client.CheckRetry = retryablehttp.HostSprayRetryPolicy()
 
 	executer := &HTTPExecutor{
+		debug:         options.Debug,
 		httpClient:    client,
 		template:      options.Template,
 		httpRequest:   options.HTTPRequest,
@@ -88,6 +94,16 @@ mainLoop:
 		}
 		e.setCustomHeaders(compiledRequest)
 		req := compiledRequest.Request
+
+		if e.debug {
+			gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID)
+			dumpedRequest, err := httputil.DumpRequest(req.Request, true)
+			if err != nil {
+				return errors.Wrap(err, "could not dump http request")
+			}
+			fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
+		}
+
 		resp, err := e.httpClient.Do(req)
 		if err != nil {
 			if resp != nil {
@@ -96,6 +112,15 @@ mainLoop:
 			return errors.Wrap(err, "could not make http request")
 		}
 
+		if e.debug {
+			gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID)
+			dumpedResponse, err := httputil.DumpResponse(resp, true)
+			if err != nil {
+				return errors.Wrap(err, "could not dump http response")
+			}
+			fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
+		}
+
 		data, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
 			io.Copy(ioutil.Discard, resp.Body)
@@ -157,6 +182,9 @@ mainLoop:
 			e.writeOutputHTTP(compiledRequest, nil, extractorResults)
 		}
 	}
+
+	gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL)
+
 	return nil
 }
 

pkg/executor/executor_dns.go

@@ -2,9 +2,12 @@ package executor
 
 import (
 	"bufio"
+	"fmt"
+	"os"
 	"sync"
 
 	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/pkg/matchers"
 	"github.com/projectdiscovery/nuclei/pkg/requests"
 	"github.com/projectdiscovery/nuclei/pkg/templates"
@@ -14,6 +17,7 @@ import (
 // DNSExecutor is a client for performing a DNS request
 // for a template.
 type DNSExecutor struct {
+	debug       bool
 	dnsClient   *retryabledns.Client
 	template    *templates.Template
 	dnsRequest  *requests.DNSRequest
@@ -31,6 +35,7 @@ var DefaultResolvers = []string{
 
 // DNSOptions contains configuration options for the DNS executor.
 type DNSOptions struct {
+	Debug      bool
 	Template   *templates.Template
 	DNSRequest *requests.DNSRequest
 	Writer     *bufio.Writer
@@ -42,6 +47,7 @@ func NewDNSExecutor(options *DNSOptions) *DNSExecutor {
 	dnsClient := retryabledns.New(DefaultResolvers, options.DNSRequest.Retries)
 
 	executer := &DNSExecutor{
+		debug:       options.Debug,
 		dnsClient:   dnsClient,
 		template:    options.Template,
 		dnsRequest:  options.DNSRequest,
@@ -67,12 +73,24 @@ func (e *DNSExecutor) ExecuteDNS(URL string) error {
 		return errors.Wrap(err, "could not make dns request")
 	}
 
+	if e.debug {
+		gologger.Infof("Dumped DNS request for %s (%s)\n\n", URL, e.template.ID)
+		fmt.Fprintf(os.Stderr, "%s\n", compiledRequest.String())
+	}
+
 	// Send the request to the target servers
 	resp, err := e.dnsClient.Do(compiledRequest)
 	if err != nil {
 		return errors.Wrap(err, "could not send dns request")
 	}
 
+	gologger.Verbosef("Sent DNS request to %s\n", "dns-request", URL)
+
+	if e.debug {
+		gologger.Infof("Dumped DNS response for %s (%s)\n\n", URL, e.template.ID)
+		fmt.Fprintf(os.Stderr, "%s\n", resp.String())
+	}
+
 	matcherCondition := e.dnsRequest.GetMatchersCondition()
 	for _, matcher := range e.dnsRequest.Matchers {
 		// Check if the matcher matched