From 8f6280dc9dcd1bebebc09ec66cb3bb79c0ef0887 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Mon, 1 Nov 2021 20:45:54 +0200 Subject: [PATCH] refactor: In case of binary data, show a hexadecimal view as well #1080 * small enhancements with regards to dumping responses --- v2/pkg/protocols/dns/request.go | 14 ++++----- v2/pkg/protocols/file/request.go | 14 ++++----- v2/pkg/protocols/headless/request.go | 11 +++---- v2/pkg/protocols/http/request.go | 43 ++++++++++++++++------------ v2/pkg/protocols/network/request.go | 41 +++++++++++++++----------- 5 files changed, 70 insertions(+), 53 deletions(-) diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index d063825f..08422b8e 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -65,22 +65,22 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - debug(event, request, domain, response.String()) + dumpResponse(event, request.options, response.String(), domain) callback(event) return nil } -func debug(event *output.InternalWrappedEvent, request *Request, domain string, response string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped DNS response for %s\n", request.options.TemplateID, domain) - +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response string, domain string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { hexDump := false - if !responsehighlighter.IsASCII(response) { + if responsehighlighter.HasBinaryContent(response) { hexDump = true response = hex.Dump([]byte(response)) } - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump)) + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, response, cliOptions.NoColor, hexDump) + gologger.Debug().Msgf("[%s] Dumped DNS response for %s\n\n%s", requestOptions.TemplateID, domain, highlightedResponse) } } diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index c7c3daee..0326de37 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -2,7 +2,6 @@ package file import ( "encoding/hex" - "fmt" "io/ioutil" "os" @@ -61,7 +60,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - debug(event, request, filePath, fileContent) + dumpResponse(event, request.options, fileContent, filePath) callback(event) }(data) @@ -76,14 +75,15 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review return nil } -func debug(event *output.InternalWrappedEvent, request *Request, filePath string, fileContent string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, fileContent string, filePath string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { hexDump := false - if !responsehighlighter.IsASCII(fileContent) { + if responsehighlighter.HasBinaryContent(fileContent) { hexDump = true fileContent = hex.Dump([]byte(fileContent)) } - logHeader := fmt.Sprintf("[%s] Dumped file request for %s\n", request.options.TemplateID, filePath) - gologger.Debug().Msgf("%s\n%s", logHeader, responsehighlighter.Highlight(event.OperatorsResult, fileContent, request.options.Options.NoColor, hexDump)) + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, fileContent, cliOptions.NoColor, hexDump) + gologger.Debug().Msgf("[%s] Dumped file request for %s\n\n%s", requestOptions.TemplateID, filePath, highlightedResponse) } } diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 1609a595..f4c0618b 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -67,15 +67,16 @@ func (request *Request) ExecuteWithResults(inputURL string, metadata, previous o event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - debug(event, request, responseBody, inputURL) + dumpResponse(event, request.options, responseBody, inputURL) callback(event) return nil } -func debug(event *output.InternalWrappedEvent, request *Request, responseBody string, input string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n", request.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor, false)) +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, responseBody string, input string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, responseBody, cliOptions.NoColor, false) + gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n\n%s", requestOptions.TemplateID, input, highlightedResponse) } } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index b11340bf..28110d38 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -511,7 +511,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta }) - debug(request, formedURL, redirectedResponse, responseContentType, event) + dumpResponse(event, request.options, redirectedResponse, formedURL, responseContentType) callback(event) return nil @@ -534,26 +534,33 @@ func (request *Request) setCustomHeaders(req *generatedRequest) { const CRLF = "\r\n" -func debug(request *Request, formedURL string, redirectedResponse []byte, responseContentType string, event *output.InternalWrappedEvent) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - hexDump := false +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, redirectedResponse []byte, formedURL string, responseContentType string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { response := string(redirectedResponse) - var headers string - if responseContentType == "" || responseContentType == "application/octet-stream" || (responseContentType == "application/x-www-form-urlencoded" && responsehighlighter.IsASCII(response)) { - hexDump = true - responseLines := strings.Split(response, CRLF) - for i, value := range responseLines { - headers += value + CRLF - if value == "" { - response = hex.Dump([]byte(strings.Join(responseLines[i+1:], ""))) - break - } - } + var highlightedResult string + if responseContentType == "application/octet-stream" || ((responseContentType == "" || responseContentType == "application/x-www-form-urlencoded") && responsehighlighter.HasBinaryContent(response)) { + highlightedResult = createResponseHexDump(event, response, cliOptions.NoColor) + } else { + highlightedResult = responsehighlighter.Highlight(event.OperatorsResult, response, cliOptions.NoColor, false) } - logMessageHeader := fmt.Sprintf("[%s] Dumped HTTP response for %s\n", request.options.TemplateID, formedURL) - gologger.Debug().Msgf("%s\n%s", logMessageHeader, responsehighlighter.Highlight(event.OperatorsResult, headers, request.options.Options.NoColor, false)) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump)) + gologger.Debug().Msgf("[%s] Dumped HTTP response for %s\n\n%s", requestOptions.TemplateID, formedURL, highlightedResult) + } +} + +func createResponseHexDump(event *output.InternalWrappedEvent, response string, noColor bool) string { + CRLFs := CRLF + CRLF + headerEndIndex := strings.Index(response, CRLFs) + len(CRLFs) + if headerEndIndex > 0 { + headers := response[0:headerEndIndex] + responseBodyHexDump := hex.Dump([]byte(response[headerEndIndex:])) + + highlightedHeaders := responsehighlighter.Highlight(event.OperatorsResult, headers, noColor, false) + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, responseBodyHexDump, noColor, true) + return fmt.Sprintf("%s\n%s", highlightedHeaders, highlightedResponse) + } else { + return responsehighlighter.Highlight(event.OperatorsResult, hex.Dump([]byte(response)), noColor, true) } } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 76135085..fa1e1b60 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" @@ -189,9 +190,8 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input request.options.Progress.IncrementRequests() if request.options.Options.Debug || request.options.Options.DebugRequests { - gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n", request.options.TemplateID, actualAddress) requestBytes := []byte(reqBuilder.String()) - gologger.Print().Msgf("%s", hex.Dump(requestBytes)) + gologger.Debug().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n%s", request.options.TemplateID, actualAddress, hex.Dump(requestBytes)) if request.options.Options.VerboseVerbose { gologger.Print().Msgf("\nCompact HEX view:\n%s", hex.EncodeToString(requestBytes)) } @@ -277,29 +277,38 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input }) } - debug(event, request, response, actualAddress) + dumpResponse(event, request.options, response, actualAddress) return nil } -func debug(event *output.InternalWrappedEvent, request *Request, response string, actualAddress string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Network response for %s\n", request.options.TemplateID, actualAddress) +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response string, actualAddress string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { requestBytes := []byte(response) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, hex.Dump(requestBytes), request.options.Options.NoColor, true)) - if request.options.Options.VerboseVerbose { - var allMatches []string - for _, namedMatch := range event.OperatorsResult.Matches { - for _, matchElement := range namedMatch { - allMatches = append(allMatches, hex.EncodeToString([]byte(matchElement))) - } - } - event.OperatorsResult.Matches["compact"] = allMatches - gologger.Print().Msgf("\nCompact HEX view:\n%s", responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), request.options.Options.NoColor, false)) + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, hex.Dump(requestBytes), cliOptions.NoColor, true) + gologger.Debug().Msgf("[%s] Dumped Network response for %s\n\n%s", requestOptions.TemplateID, actualAddress, highlightedResponse) + + if cliOptions.VerboseVerbose { + displayCompactHexView(event, response, cliOptions.NoColor) } } } +func displayCompactHexView(event *output.InternalWrappedEvent, response string, noColor bool) { + operatorsResult := event.OperatorsResult + if operatorsResult != nil { + var allMatches []string + for _, namedMatch := range operatorsResult.Matches { + for _, matchElement := range namedMatch { + allMatches = append(allMatches, hex.EncodeToString([]byte(matchElement))) + } + } + tempOperatorResult := &operators.Result{Matches: map[string][]string{"matchesInHex": allMatches}} + gologger.Print().Msgf("\nCompact HEX view:\n%s", responsehighlighter.Highlight(tempOperatorResult, hex.EncodeToString([]byte(response)), noColor, false)) + } +} + // getAddress returns the address of the host to make request to func getAddress(toTest string) (string, error) { if strings.Contains(toTest, "://") {