daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue 1705 save responses on disk (#1727) 

* save response on disk

* lint error check

* store raw request/response

* lint error fix

* file path

* mock test fix

* readme update

* .txt extension

Co-authored-by: sandeep <sandeep@projectdiscovery.io>
dev
Sami 2022-04-01 14:29:02 -05:00 committed by GitHub
parent 36355908e8
commit 301307bb77
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 179 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -187,7 +187,9 @@ DEBUG:
-debug show all requests and responses
-debug-req show all sent requests
-debug-resp show all received responses
-p, -proxy string[] List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)
-sresp, -store-resp store all request/response passed through nuclei to output directory
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
-p, -proxy string[] list of http/socks5 proxy to use (comma separated or file input)
-pi, -proxy-internal proxy all internal requests
-tlog, -trace-log string file to write sent requests trace log
-elog, -error-log string file to write sent requests error log

8
v2/cmd/nuclei/main.go
View File

@ -186,9 +186,11 @@ on extensive configurability, massive extensibility and ease of use.`)
createGroup(flagSet, "debug", "Debug",
flagSet.BoolVar(&options.Debug, "debug", false, "show all requests and responses"),
flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"),
flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"),
flagSet.NormalizedOriginalStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)"),
flagSet.BoolVarP(&options.DebugRequests, "debug-req", "dreq", false, "show all sent requests"),
flagSet.BoolVarP(&options.DebugResponse, "debug-resp", "dresp", false, "show all received responses"),
flagSet.BoolVarP(&options.StoreResponse, "store-resp", "sresp", false, "store all request/response passed through nuclei to output directory"),
flagSet.StringVarP(&options.StoreResponseDir, "store-resp-dir", "srd", "output", "store all request/response passed through nuclei to custom directory"),
flagSet.NormalizedOriginalStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "list of http/socks5 proxy to use (comma separated or file input)"),
flagSet.BoolVarP(&options.ProxyInternal, "proxy-internal", "pi", false, "proxy all internal requests"),
flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"),
flagSet.StringVarP(&options.ErrorLogFile, "error-log", "elog", "", "file to write sent requests error log"),

5
v2/internal/runner/options.go
View File

@ -57,7 +57,10 @@ func ParseOptions(options *types.Options) {
gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", configuration.TemplateVersion, configuration.TemplatesDirectory)
os.Exit(0)
}
if options.StoreResponseDir != "" && !options.StoreResponse {
gologger.Debug().Msgf("Store response directory specified, enabling \"str\" flag automatically\n")
options.StoreResponse = true
}
// Validate the options passed by the user and if any
// invalid options have been used, exit.
if err := validateOptions(options); err != nil {

2
v2/internal/runner/runner.go
View File

@ -158,7 +158,7 @@ func New(options *types.Options) (*Runner, error) {
runner.hmapInputProvider = hmapInput
// Create the output file if asked
outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.MatcherStatus, options.Output, options.TraceLogFile, options.ErrorLogFile)
outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.MatcherStatus, options.StoreResponse, options.Output, options.TraceLogFile, options.ErrorLogFile, options.StoreResponseDir)
if err != nil {
return nil, errors.Wrap(err, "could not create output file")
}

88
v2/pkg/output/output.go
View File

@ -1,9 +1,12 @@
package output
import (
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/pkg/errors"
@ -11,6 +14,8 @@ import (
jsoniter "github.com/json-iterator/go"
"github.com/logrusorgru/aurora"
"github.com/projectdiscovery/fileutil"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/interactsh/pkg/server"
"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
@ -32,20 +37,24 @@ type Writer interface {
WriteFailure(event InternalEvent) error
// Request logs a request in the trace log
Request(templateID, url, requestType string, err error)
// WriteStoreDebugData writes the request/response debug data to file
WriteStoreDebugData(host, templateID, eventType string, data string)
}
// StandardWriter is a writer writing output to file and screen for results.
type StandardWriter struct {
json bool
jsonReqResp bool
noTimestamp bool
noMetadata bool
matcherStatus bool
aurora aurora.Aurora
outputFile io.WriteCloser
traceFile io.WriteCloser
errorFile io.WriteCloser
severityColors func(severity.Severity) string
json bool
jsonReqResp bool
noTimestamp bool
noMetadata bool
matcherStatus bool
aurora aurora.Aurora
outputFile io.WriteCloser
traceFile io.WriteCloser
errorFile io.WriteCloser
severityColors func(severity.Severity) string
storeResponse bool
storeResponseDir string
}
var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
@ -112,7 +121,7 @@ type ResultEvent struct {
}
// NewStandardWriter creates a new output writer based on user configurations
func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp, MatcherStatus bool, file, traceFile string, errorFile string) (*StandardWriter, error) {
func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp, MatcherStatus, storeResponse bool, file, traceFile string, errorFile string, storeResponseDir string) (*StandardWriter, error) {
auroraColorizer := aurora.NewAurora(colors)
var outputFile io.WriteCloser
@ -139,17 +148,25 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp, Match
}
errorOutput = output
}
// Try to create output folder if it doesn't exist
if storeResponse && !fileutil.FolderExists(storeResponseDir) {
if err := fileutil.CreateFolder(storeResponseDir); err != nil {
gologger.Fatal().Msgf("Could not create output directory '%s': %s\n", storeResponseDir, err)
}
}
writer := &StandardWriter{
json: json,
jsonReqResp: jsonReqResp,
noMetadata: noMetadata,
matcherStatus: MatcherStatus,
noTimestamp: noTimestamp,
aurora: auroraColorizer,
outputFile: outputFile,
traceFile: traceOutput,
errorFile: errorOutput,
severityColors: colorizer.New(auroraColorizer),
json: json,
jsonReqResp: jsonReqResp,
noMetadata: noMetadata,
matcherStatus: MatcherStatus,
noTimestamp: noTimestamp,
aurora: auroraColorizer,
outputFile: outputFile,
traceFile: traceOutput,
errorFile: errorOutput,
severityColors: colorizer.New(auroraColorizer),
storeResponse: storeResponse,
storeResponseDir: storeResponseDir,
}
return writer, nil
}
@ -178,6 +195,7 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
}
_, _ = os.Stdout.Write(data)
_, _ = os.Stdout.Write([]byte("\n"))
if w.outputFile != nil {
if !w.json {
data = decolorizerRegex.ReplaceAll(data, []byte(""))
@ -264,3 +282,31 @@ func (w *StandardWriter) WriteFailure(event InternalEvent) error {
}
return w.Write(data)
}
func sanitizeFileName(fileName string) string {
fileName = strings.ReplaceAll(fileName, "http:", "")
fileName = strings.ReplaceAll(fileName, "https:", "")
fileName = strings.ReplaceAll(fileName, "/", "_")
fileName = strings.ReplaceAll(fileName, "\\", "_")
fileName = strings.ReplaceAll(fileName, "-", "_")
fileName = strings.ReplaceAll(fileName, ".", "_")
fileName = strings.TrimPrefix(fileName, "__")
return fileName
}
func (w *StandardWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {
if w.storeResponse {
filename := sanitizeFileName(fmt.Sprintf("%s_%s", host, templateID))
subFolder := filepath.Join(w.storeResponseDir, sanitizeFileName(eventType))
if !fileutil.FolderExists(subFolder) {
_ = fileutil.CreateFolder(subFolder)
}
filename = filepath.Join(subFolder, fmt.Sprintf("%s.txt", filename))
f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, os.ModePerm)
if err != nil {
fmt.Print(err)
return
}
_, _ = f.WriteString(fmt.Sprintln(data))
f.Close()
}
}

6
v2/pkg/output/output_test.go
View File

@ -11,7 +11,7 @@ import (
func TestStandardWriterRequest(t *testing.T) {
t.Run("WithoutTraceAndError", func(t *testing.T) {
w, err := NewStandardWriter(false, false, false, false, false, false, "", "", "")
w, err := NewStandardWriter(false, false, false, false, false, false, false, "", "", "", "")
require.NoError(t, err)
require.NotPanics(t, func() {
w.Request("path", "input", "http", nil)
@ -23,7 +23,7 @@ func TestStandardWriterRequest(t *testing.T) {
traceWriter := &testWriteCloser{}
errorWriter := &testWriteCloser{}
w, err := NewStandardWriter(false, false, false, false, false, false, "", "", "")
w, err := NewStandardWriter(false, false, false, false, false, false, false, "", "", "", "")
w.traceFile = traceWriter
w.errorFile = errorWriter
require.NoError(t, err)
@ -36,7 +36,7 @@ func TestStandardWriterRequest(t *testing.T) {
t.Run("ErrorWithWrappedError", func(t *testing.T) {
errorWriter := &testWriteCloser{}
w, err := NewStandardWriter(false, false, false, false, false, false, "", "", "")
w, err := NewStandardWriter(false, false, false, false, false, false, false, "", "", "", "")
w.errorFile = errorWriter
require.NoError(t, err)
w.Request(

29
v2/pkg/protocols/dns/request.go
View File

@ -2,6 +2,7 @@ package dns
import (
"encoding/hex"
"fmt"
"net/url"
"github.com/pkg/errors"
@ -55,9 +56,15 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", request.options.TemplateID, domain, varErr)
return nil
}
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Str("domain", domain).Msgf("[%s] Dumped DNS request for %s", request.options.TemplateID, domain)
gologger.Print().Msgf("%s", requestString)
if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.StoreResponse {
msg := fmt.Sprintf("[%s] Dumped DNS request for %s", request.options.TemplateID, domain)
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Str("domain", domain).Msgf(msg)
gologger.Print().Msgf("%s", requestString)
}
if request.options.Options.StoreResponse {
request.options.Output.WriteStoreDebugData(domain, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, requestString))
}
}
// Send the request to the target servers
@ -91,7 +98,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
// TODO: dynamic values are not supported yet
dumpResponse(event, request.options, response.String(), domain)
dumpResponse(event, request, response.String(), domain)
if request.Trace {
dumpTraceData(event, request.options, traceToString(traceData, true), domain)
}
@ -100,16 +107,22 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
return nil
}
func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response, domain string) {
cliOptions := requestOptions.Options
if cliOptions.Debug || cliOptions.DebugResponse {
func dumpResponse(event *output.InternalWrappedEvent, request *Request, response, domain string) {
cliOptions := request.options.Options
if cliOptions.Debug || cliOptions.DebugResponse || cliOptions.StoreResponse {
hexDump := false
if responsehighlighter.HasBinaryContent(response) {
hexDump = true
response = hex.Dump([]byte(response))
}
highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, response, cliOptions.NoColor, hexDump)
gologger.Debug().Msgf("[%s] Dumped DNS response for %s\n\n%s", requestOptions.TemplateID, domain, highlightedResponse)
msg := fmt.Sprintf("[%s] Dumped DNS response for %s\n\n%s", request.options.TemplateID, domain, highlightedResponse)
if cliOptions.Debug || cliOptions.DebugResponse {
gologger.Debug().Msg(msg)
}
if cliOptions.StoreResponse {
request.options.Output.WriteStoreDebugData(domain, request.options.TemplateID, request.Type().String(), msg)
}
}
}

42
v2/pkg/protocols/http/request.go
View File

@ -65,9 +65,15 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern
if err != nil {
return err
}
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
gologger.Print().Msgf("%s", string(dumpedRequest))
if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.StoreResponse {
msg := fmt.Sprintf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Msg(msg)
gologger.Print().Msgf("%s", string(dumpedRequest))
}
if request.options.Options.StoreResponse {
request.options.Output.WriteStoreDebugData(reqURL, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, dumpedRequest))
}
}
previous["request"] = string(dumpedRequest)
@ -426,9 +432,16 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
return dumpError
}
dumpedRequestString := string(dumpedRequest)
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
gologger.Print().Msgf("%s", dumpedRequestString)
if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.StoreResponse {
msg := fmt.Sprintf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Msg(msg)
gologger.Print().Msgf("%s", dumpedRequestString)
}
if request.options.Options.StoreResponse {
request.options.Output.WriteStoreDebugData(reqURL, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, dumpedRequestString))
}
}
}
@ -578,7 +591,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
responseContentType := resp.Header.Get("Content-Type")
isResponseTruncated := len(gotData) >= request.MaxSize
dumpResponse(event, request.options, response.fullResponse, formedURL, responseContentType, isResponseTruncated)
dumpResponse(event, request, response.fullResponse, formedURL, responseContentType, isResponseTruncated, reqURL)
callback(event)
}
@ -636,9 +649,9 @@ func (request *Request) setCustomHeaders(req *generatedRequest) {
const CRLF = "\r\n"
func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, redirectedResponse []byte, formedURL string, responseContentType string, isResponseTruncated bool) {
cliOptions := requestOptions.Options
if cliOptions.Debug || cliOptions.DebugResponse {
func dumpResponse(event *output.InternalWrappedEvent, request *Request, redirectedResponse []byte, formedURL string, responseContentType string, isResponseTruncated bool, reqURL string) {
cliOptions := request.options.Options
if cliOptions.Debug || cliOptions.DebugResponse || cliOptions.StoreResponse {
response := string(redirectedResponse)
var highlightedResult string
@ -652,8 +665,13 @@ func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.
if isResponseTruncated {
msg = "[%s] Dumped HTTP response (Truncated) %s\n\n%s"
}
gologger.Debug().Msgf(msg, requestOptions.TemplateID, formedURL, highlightedResult)
fMsg := fmt.Sprintf(msg, request.options.TemplateID, formedURL, highlightedResult)
if cliOptions.Debug || cliOptions.DebugResponse {
gologger.Debug().Msg(fMsg)
}
if cliOptions.StoreResponse {
request.options.Output.WriteStoreDebugData(reqURL, request.options.TemplateID, request.Type().String(), fMsg)
}
}
}

28
v2/pkg/protocols/network/request.go
View File

@ -3,6 +3,7 @@ package network
import (
"context"
"encoding/hex"
"fmt"
"io"
"net"
"net/url"
@ -185,9 +186,15 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
}
request.options.Progress.IncrementRequests()
if request.options.Options.Debug || request.options.Options.DebugRequests {
if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.StoreResponse{
requestBytes := []byte(reqBuilder.String())
gologger.Debug().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n%s", request.options.TemplateID, actualAddress, hex.Dump(requestBytes))
msg := fmt.Sprintf("[%s] Dumped Network request for %s\n%s", request.options.TemplateID, actualAddress, hex.Dump(requestBytes))
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Str("address", actualAddress).Msg(msg)
}
if request.options.Options.StoreResponse{
request.options.Output.WriteStoreDebugData(address, request.options.TemplateID, request.Type().String(), msg)
}
if request.options.Options.VerboseVerbose {
gologger.Print().Msgf("\nCompact HEX view:\n%s", hex.EncodeToString(requestBytes))
}
@ -282,18 +289,23 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
event.UsesInteractsh = true
}
dumpResponse(event, request.options, response, actualAddress)
dumpResponse(event, request, response, actualAddress, address)
return nil
}
func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response string, actualAddress string) {
cliOptions := requestOptions.Options
if cliOptions.Debug || cliOptions.DebugResponse {
func dumpResponse(event *output.InternalWrappedEvent, request *Request, response string, actualAddress, address string) {
cliOptions := request.options.Options
if cliOptions.Debug || cliOptions.DebugResponse || cliOptions.StoreResponse{
requestBytes := []byte(response)
highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, hex.Dump(requestBytes), cliOptions.NoColor, true)
gologger.Debug().Msgf("[%s] Dumped Network response for %s\n\n%s", requestOptions.TemplateID, actualAddress, highlightedResponse)
msg := fmt.Sprintf("[%s] Dumped Network response for %s\n\n", request.options.TemplateID, actualAddress)
if cliOptions.Debug || cliOptions.DebugResponse {
gologger.Debug().Msg(fmt.Sprintf("%s%s", msg, highlightedResponse))
}
if cliOptions.StoreResponse{
request.options.Output.WriteStoreDebugData(address, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s%s", msg, hex.Dump(requestBytes)))
}
if cliOptions.VerboseVerbose {
displayCompactHexView(event, response, cliOptions.NoColor)
}

21
v2/pkg/protocols/ssl/ssl.go
View File

@ -3,6 +3,7 @@ package ssl
import (
"context"
"crypto/tls"
"fmt"
"net"
"net/url"
"strings"
@ -182,8 +183,14 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous
requestOptions.Output.Request(requestOptions.TemplateID, address, request.Type().String(), err)
gologger.Verbose().Msgf("Sent SSL request to %s", address)
if requestOptions.Options.Debug || requestOptions.Options.DebugRequests {
gologger.Debug().Str("address", input).Msgf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input)
if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse {
msg := fmt.Sprintf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input)
if requestOptions.Options.Debug || requestOptions.Options.DebugRequests {
gologger.Debug().Str("address", input).Msg(msg)
}
if requestOptions.Options.StoreResponse {
request.options.Output.WriteStoreDebugData(input, request.options.TemplateID, request.Type().String(), msg)
}
}
var (
@ -228,9 +235,15 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous
data["ip"] = request.dialer.GetDialedIP(hostname)
event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse)
if requestOptions.Options.Debug || requestOptions.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input)
if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse {
msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input)
if requestOptions.Options.Debug || requestOptions.Options.DebugResponse {
gologger.Debug().Msg(msg)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, requestOptions.Options.NoColor, false))
}
if requestOptions.Options.StoreResponse {
request.options.Output.WriteStoreDebugData(input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, jsonDataString))
}
}
callback(event)
return nil

3
v2/pkg/testutils/testutils.go
View File

@ -136,6 +136,9 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro
func (m *MockOutputWriter) WriteFailure(result output.InternalEvent) error {
return nil
}
func (m *MockOutputWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {
}
type MockProgressClient struct{}

6
v2/pkg/types/types.go
View File

@ -207,7 +207,11 @@ type Options struct {
// Use ZTLS library
ZTLS bool
// EnablePprof enables exposing pprof runtime information with a webserver.
EnablePprof bool
EnablePprof bool
// StoreResponse stores received response to output directory
StoreResponse bool
// StoreResponseDir stores received response to custom directory
StoreResponseDir string
}
func (options *Options) AddVarPayload(key string, value interface{}) {