Issue 1705 save responses on disk (#1727)

* save response on disk

* lint error check

* store raw request/response

* lint error fix

* file path

* mock test fix

* readme update

* .txt extension

Co-authored-by: sandeep <sandeep@projectdiscovery.io>

README.md

@@ -187,7 +187,9 @@ DEBUG:
    -debug                    show all requests and responses
    -debug-req                show all sent requests
    -debug-resp               show all received responses
-   -p, -proxy string[]       List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)
+   -sresp, -store-resp           store all request/response passed through nuclei to output directory
+   -srd, -store-resp-dir string  store all request/response passed through nuclei to custom directory (default "output")
+   -p, -proxy string[]           list of http/socks5 proxy to use (comma separated or file input)
    -pi, -proxy-internal      proxy all internal requests
    -tlog, -trace-log string  file to write sent requests trace log
    -elog, -error-log string  file to write sent requests error log

v2/cmd/nuclei/main.go

@@ -186,9 +186,11 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	createGroup(flagSet, "debug", "Debug",
 		flagSet.BoolVar(&options.Debug, "debug", false, "show all requests and responses"),
-		flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"),
-		flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"),
-		flagSet.NormalizedOriginalStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)"),
+		flagSet.BoolVarP(&options.DebugRequests, "debug-req", "dreq", false, "show all sent requests"),
+		flagSet.BoolVarP(&options.DebugResponse, "debug-resp", "dresp", false, "show all received responses"),
+		flagSet.BoolVarP(&options.StoreResponse, "store-resp", "sresp", false, "store all request/response passed through nuclei to output directory"),
+		flagSet.StringVarP(&options.StoreResponseDir, "store-resp-dir", "srd", "output", "store all request/response passed through nuclei to custom directory"),
+		flagSet.NormalizedOriginalStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "list of http/socks5 proxy to use (comma separated or file input)"),
 		flagSet.BoolVarP(&options.ProxyInternal, "proxy-internal", "pi", false, "proxy all internal requests"),
 		flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"),
 		flagSet.StringVarP(&options.ErrorLogFile, "error-log", "elog", "", "file to write sent requests error log"),

v2/internal/runner/options.go

@@ -57,7 +57,10 @@ func ParseOptions(options *types.Options) {
 		gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", configuration.TemplateVersion, configuration.TemplatesDirectory)
 		os.Exit(0)
 	}
-
+	if options.StoreResponseDir != "" && !options.StoreResponse {
+		gologger.Debug().Msgf("Store response directory specified, enabling \"str\" flag automatically\n")
+		options.StoreResponse = true
+	}
 	// Validate the options passed by the user and if any
 	// invalid options have been used, exit.
 	if err := validateOptions(options); err != nil {

v2/internal/runner/runner.go

@@ -158,7 +158,7 @@ func New(options *types.Options) (*Runner, error) {
 	runner.hmapInputProvider = hmapInput
 
 	// Create the output file if asked
-	outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.MatcherStatus, options.Output, options.TraceLogFile, options.ErrorLogFile)
+	outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.MatcherStatus, options.StoreResponse, options.Output, options.TraceLogFile, options.ErrorLogFile, options.StoreResponseDir)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create output file")
 	}

v2/pkg/output/output.go

@@ -1,9 +1,12 @@
 package output
 
 import (
+	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -11,6 +14,8 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"github.com/logrusorgru/aurora"
 
+	"github.com/projectdiscovery/fileutil"
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/interactsh/pkg/server"
 	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model"
@@ -32,6 +37,8 @@ type Writer interface {
 	WriteFailure(event InternalEvent) error
 	// Request logs a request in the trace log
 	Request(templateID, url, requestType string, err error)
+	//  WriteStoreDebugData writes the request/response debug data to file
+	WriteStoreDebugData(host, templateID, eventType string, data string)
 }
 
 // StandardWriter is a writer writing output to file and screen for results.
@@ -46,6 +53,8 @@ type StandardWriter struct {
 	traceFile        io.WriteCloser
 	errorFile        io.WriteCloser
 	severityColors   func(severity.Severity) string
+	storeResponse    bool
+	storeResponseDir string
 }
 
 var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
@@ -112,7 +121,7 @@ type ResultEvent struct {
 }
 
 // NewStandardWriter creates a new output writer based on user configurations
-func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp, MatcherStatus bool, file, traceFile string, errorFile string) (*StandardWriter, error) {
+func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp, MatcherStatus, storeResponse bool, file, traceFile string, errorFile string, storeResponseDir string) (*StandardWriter, error) {
 	auroraColorizer := aurora.NewAurora(colors)
 
 	var outputFile io.WriteCloser
@@ -139,6 +148,12 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp, Match
 		}
 		errorOutput = output
 	}
+	// Try to create output folder if it doesn't exist
+	if storeResponse && !fileutil.FolderExists(storeResponseDir) {
+		if err := fileutil.CreateFolder(storeResponseDir); err != nil {
+			gologger.Fatal().Msgf("Could not create output directory '%s': %s\n", storeResponseDir, err)
+		}
+	}
 	writer := &StandardWriter{
 		json:             json,
 		jsonReqResp:      jsonReqResp,
@@ -150,6 +165,8 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp, Match
 		traceFile:        traceOutput,
 		errorFile:        errorOutput,
 		severityColors:   colorizer.New(auroraColorizer),
+		storeResponse:    storeResponse,
+		storeResponseDir: storeResponseDir,
 	}
 	return writer, nil
 }
@@ -178,6 +195,7 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
 	}
 	_, _ = os.Stdout.Write(data)
 	_, _ = os.Stdout.Write([]byte("\n"))
+
 	if w.outputFile != nil {
 		if !w.json {
 			data = decolorizerRegex.ReplaceAll(data, []byte(""))
@@ -264,3 +282,31 @@ func (w *StandardWriter) WriteFailure(event InternalEvent) error {
 	}
 	return w.Write(data)
 }
+func sanitizeFileName(fileName string) string {
+	fileName = strings.ReplaceAll(fileName, "http:", "")
+	fileName = strings.ReplaceAll(fileName, "https:", "")
+	fileName = strings.ReplaceAll(fileName, "/", "_")
+	fileName = strings.ReplaceAll(fileName, "\\", "_")
+	fileName = strings.ReplaceAll(fileName, "-", "_")
+	fileName = strings.ReplaceAll(fileName, ".", "_")
+	fileName = strings.TrimPrefix(fileName, "__")
+	return fileName
+}
+func (w *StandardWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {
+	if w.storeResponse {
+		filename := sanitizeFileName(fmt.Sprintf("%s_%s", host, templateID))
+		subFolder := filepath.Join(w.storeResponseDir, sanitizeFileName(eventType))
+		if !fileutil.FolderExists(subFolder) {
+			_ = fileutil.CreateFolder(subFolder)
+		}
+		filename = filepath.Join(subFolder, fmt.Sprintf("%s.txt", filename))
+		f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, os.ModePerm)
+		if err != nil {
+			fmt.Print(err)
+			return
+		}
+		_, _ = f.WriteString(fmt.Sprintln(data))
+		f.Close()
+	}
+
+}

v2/pkg/output/output_test.go

@@ -11,7 +11,7 @@ import (
 
 func TestStandardWriterRequest(t *testing.T) {
 	t.Run("WithoutTraceAndError", func(t *testing.T) {
-		w, err := NewStandardWriter(false, false, false, false, false, false, "", "", "")
+		w, err := NewStandardWriter(false, false, false, false, false, false, false, "", "", "", "")
 		require.NoError(t, err)
 		require.NotPanics(t, func() {
 			w.Request("path", "input", "http", nil)
@@ -23,7 +23,7 @@ func TestStandardWriterRequest(t *testing.T) {
 		traceWriter := &testWriteCloser{}
 		errorWriter := &testWriteCloser{}
 
-		w, err := NewStandardWriter(false, false, false, false, false, false, "", "", "")
+		w, err := NewStandardWriter(false, false, false, false, false, false, false, "", "", "", "")
 		w.traceFile = traceWriter
 		w.errorFile = errorWriter
 		require.NoError(t, err)
@@ -36,7 +36,7 @@ func TestStandardWriterRequest(t *testing.T) {
 	t.Run("ErrorWithWrappedError", func(t *testing.T) {
 		errorWriter := &testWriteCloser{}
 
-		w, err := NewStandardWriter(false, false, false, false, false, false, "", "", "")
+		w, err := NewStandardWriter(false, false, false, false, false, false, false, "", "", "", "")
 		w.errorFile = errorWriter
 		require.NoError(t, err)
 		w.Request(

v2/pkg/protocols/dns/request.go

@@ -2,6 +2,7 @@ package dns
 
 import (
 	"encoding/hex"
+	"fmt"
 	"net/url"
 
 	"github.com/pkg/errors"
@@ -55,10 +56,16 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
 		gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", request.options.TemplateID, domain, varErr)
 		return nil
 	}
+	if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.StoreResponse {
+		msg := fmt.Sprintf("[%s] Dumped DNS request for %s", request.options.TemplateID, domain)
 		if request.options.Options.Debug || request.options.Options.DebugRequests {
-		gologger.Info().Str("domain", domain).Msgf("[%s] Dumped DNS request for %s", request.options.TemplateID, domain)
+			gologger.Info().Str("domain", domain).Msgf(msg)
 			gologger.Print().Msgf("%s", requestString)
 		}
+		if request.options.Options.StoreResponse {
+			request.options.Output.WriteStoreDebugData(domain, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, requestString))
+		}
+	}
 
 	// Send the request to the target servers
 	response, err := dnsClient.Do(compiledRequest)
@@ -91,7 +98,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
 	event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
 	// TODO: dynamic values are not supported yet
 
-	dumpResponse(event, request.options, response.String(), domain)
+	dumpResponse(event, request, response.String(), domain)
 	if request.Trace {
 		dumpTraceData(event, request.options, traceToString(traceData, true), domain)
 	}
@@ -100,16 +107,22 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
 	return nil
 }
 
-func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response, domain string) {
-	cliOptions := requestOptions.Options
-	if cliOptions.Debug || cliOptions.DebugResponse {
+func dumpResponse(event *output.InternalWrappedEvent, request *Request, response, domain string) {
+	cliOptions := request.options.Options
+	if cliOptions.Debug || cliOptions.DebugResponse || cliOptions.StoreResponse {
 		hexDump := false
 		if responsehighlighter.HasBinaryContent(response) {
 			hexDump = true
 			response = hex.Dump([]byte(response))
 		}
 		highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, response, cliOptions.NoColor, hexDump)
-		gologger.Debug().Msgf("[%s] Dumped DNS response for %s\n\n%s", requestOptions.TemplateID, domain, highlightedResponse)
+		msg := fmt.Sprintf("[%s] Dumped DNS response for %s\n\n%s", request.options.TemplateID, domain, highlightedResponse)
+		if cliOptions.Debug || cliOptions.DebugResponse {
+			gologger.Debug().Msg(msg)
+		}
+		if cliOptions.StoreResponse {
+			request.options.Output.WriteStoreDebugData(domain, request.options.TemplateID, request.Type().String(), msg)
+		}
 	}
 }
 

v2/pkg/protocols/http/request.go

@@ -65,10 +65,16 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern
 	if err != nil {
 		return err
 	}
+	if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.StoreResponse {
+		msg := fmt.Sprintf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
 		if request.options.Options.Debug || request.options.Options.DebugRequests {
-		gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
+			gologger.Info().Msg(msg)
 			gologger.Print().Msgf("%s", string(dumpedRequest))
 		}
+		if request.options.Options.StoreResponse {
+			request.options.Output.WriteStoreDebugData(reqURL, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, dumpedRequest))
+		}
+	}
 	previous["request"] = string(dumpedRequest)
 
 	// Pre-Generate requests
@@ -426,10 +432,17 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 			return dumpError
 		}
 		dumpedRequestString := string(dumpedRequest)
+		if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.StoreResponse {
+			msg := fmt.Sprintf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
+
 			if request.options.Options.Debug || request.options.Options.DebugRequests {
-			gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
+				gologger.Info().Msg(msg)
 				gologger.Print().Msgf("%s", dumpedRequestString)
 			}
+			if request.options.Options.StoreResponse {
+				request.options.Output.WriteStoreDebugData(reqURL, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, dumpedRequestString))
+			}
+		}
 	}
 
 	// use request url as matched url if empty
@@ -578,7 +591,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 
 		responseContentType := resp.Header.Get("Content-Type")
 		isResponseTruncated := len(gotData) >= request.MaxSize
-		dumpResponse(event, request.options, response.fullResponse, formedURL, responseContentType, isResponseTruncated)
+		dumpResponse(event, request, response.fullResponse, formedURL, responseContentType, isResponseTruncated, reqURL)
 
 		callback(event)
 	}
@@ -636,9 +649,9 @@ func (request *Request) setCustomHeaders(req *generatedRequest) {
 
 const CRLF = "\r\n"
 
-func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, redirectedResponse []byte, formedURL string, responseContentType string, isResponseTruncated bool) {
-	cliOptions := requestOptions.Options
-	if cliOptions.Debug || cliOptions.DebugResponse {
+func dumpResponse(event *output.InternalWrappedEvent, request *Request, redirectedResponse []byte, formedURL string, responseContentType string, isResponseTruncated bool, reqURL string) {
+	cliOptions := request.options.Options
+	if cliOptions.Debug || cliOptions.DebugResponse || cliOptions.StoreResponse {
 		response := string(redirectedResponse)
 
 		var highlightedResult string
@@ -652,8 +665,13 @@ func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.
 		if isResponseTruncated {
 			msg = "[%s] Dumped HTTP response (Truncated) %s\n\n%s"
 		}
-
-		gologger.Debug().Msgf(msg, requestOptions.TemplateID, formedURL, highlightedResult)
+		fMsg := fmt.Sprintf(msg, request.options.TemplateID, formedURL, highlightedResult)
+		if cliOptions.Debug || cliOptions.DebugResponse {
+			gologger.Debug().Msg(fMsg)
+		}
+		if cliOptions.StoreResponse {
+			request.options.Output.WriteStoreDebugData(reqURL, request.options.TemplateID, request.Type().String(), fMsg)
+		}
 	}
 }
 

v2/pkg/protocols/network/request.go

@@ -3,6 +3,7 @@ package network
 import (
 	"context"
 	"encoding/hex"
+	"fmt"
 	"io"
 	"net"
 	"net/url"
@@ -185,9 +186,15 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 	}
 	request.options.Progress.IncrementRequests()
 
-	if request.options.Options.Debug || request.options.Options.DebugRequests {
+	if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.StoreResponse{
 		requestBytes := []byte(reqBuilder.String())
-		gologger.Debug().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n%s", request.options.TemplateID, actualAddress, hex.Dump(requestBytes))
+		msg := fmt.Sprintf("[%s] Dumped Network request for %s\n%s", request.options.TemplateID, actualAddress, hex.Dump(requestBytes))
+		if request.options.Options.Debug || request.options.Options.DebugRequests {
+		gologger.Info().Str("address", actualAddress).Msg(msg)
+		}
+		if request.options.Options.StoreResponse{
+		request.options.Output.WriteStoreDebugData(address, request.options.TemplateID, request.Type().String(), msg)
+		}
 		if request.options.Options.VerboseVerbose {
 			gologger.Print().Msgf("\nCompact HEX view:\n%s", hex.EncodeToString(requestBytes))
 		}
@@ -282,18 +289,23 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 		event.UsesInteractsh = true
 	}
 
-	dumpResponse(event, request.options, response, actualAddress)
+	dumpResponse(event, request, response, actualAddress, address)
 
 	return nil
 }
 
-func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response string, actualAddress string) {
-	cliOptions := requestOptions.Options
-	if cliOptions.Debug || cliOptions.DebugResponse {
+func dumpResponse(event *output.InternalWrappedEvent, request *Request, response string, actualAddress, address string) {
+	cliOptions := request.options.Options
+	if cliOptions.Debug || cliOptions.DebugResponse || cliOptions.StoreResponse{
 		requestBytes := []byte(response)
 		highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, hex.Dump(requestBytes), cliOptions.NoColor, true)
-		gologger.Debug().Msgf("[%s] Dumped Network response for %s\n\n%s", requestOptions.TemplateID, actualAddress, highlightedResponse)
-
+		msg := fmt.Sprintf("[%s] Dumped Network response for %s\n\n", request.options.TemplateID, actualAddress)
+		if cliOptions.Debug || cliOptions.DebugResponse {
+		gologger.Debug().Msg(fmt.Sprintf("%s%s", msg, highlightedResponse))
+		}
+		if cliOptions.StoreResponse{
+		request.options.Output.WriteStoreDebugData(address, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s%s", msg, hex.Dump(requestBytes)))
+		}
 		if cliOptions.VerboseVerbose {
 			displayCompactHexView(event, response, cliOptions.NoColor)
 		}

v2/pkg/protocols/ssl/ssl.go

@@ -3,6 +3,7 @@ package ssl
 import (
 	"context"
 	"crypto/tls"
+	"fmt"
 	"net"
 	"net/url"
 	"strings"
@@ -182,8 +183,14 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous
 	requestOptions.Output.Request(requestOptions.TemplateID, address, request.Type().String(), err)
 	gologger.Verbose().Msgf("Sent SSL request to %s", address)
 
+	if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse {
+		msg := fmt.Sprintf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input)
 		if requestOptions.Options.Debug || requestOptions.Options.DebugRequests {
-		gologger.Debug().Str("address", input).Msgf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input)
+			gologger.Debug().Str("address", input).Msg(msg)
+		}
+		if requestOptions.Options.StoreResponse {
+			request.options.Output.WriteStoreDebugData(input, request.options.TemplateID, request.Type().String(), msg)
+		}
 	}
 
 	var (
@@ -228,10 +235,16 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous
 	data["ip"] = request.dialer.GetDialedIP(hostname)
 
 	event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse)
+	if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse {
+		msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input)
 		if requestOptions.Options.Debug || requestOptions.Options.DebugResponse {
-		gologger.Debug().Msgf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input)
+		gologger.Debug().Msg(msg)
 		gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, requestOptions.Options.NoColor, false))
 		}
+		if requestOptions.Options.StoreResponse {
+		request.options.Output.WriteStoreDebugData(input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, jsonDataString))
+		}
+	}
 	callback(event)
 	return nil
 }

v2/pkg/testutils/testutils.go

@@ -136,6 +136,9 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro
 func (m *MockOutputWriter) WriteFailure(result output.InternalEvent) error {
 	return nil
 }
+func (m *MockOutputWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {
+
+}
 
 type MockProgressClient struct{}
 

v2/pkg/types/types.go

@@ -208,6 +208,10 @@ type Options struct {
 	ZTLS bool
 	// EnablePprof enables exposing pprof runtime information with a webserver.
 	EnablePprof      bool
+	// StoreResponse stores received response to output directory
+	StoreResponse    bool
+	// StoreResponseDir stores received response to custom directory
+	StoreResponseDir string
 }
 
 func (options *Options) AddVarPayload(key string, value interface{}) {