feat: In case of binary data, show a hexadecimal view as well #1080

dev
forgedhallpass 2021-10-30 13:17:47 +03:00
parent e0afa2cee4
commit 04e3c0165a
9 changed files with 303 additions and 50 deletions

View File

@ -175,10 +175,3 @@ func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags
currentFlag.Group(groupName)
}
}
/*
HacktoberFest update: Below, you can find our ticket recommendations. Tasks with the "good first issue" label are suitable for first time contributors. If you have other ideas, or need help with getting started, join our Discord channel or reach out to @forgedhallpass.
https://github.com/issues?q=is%3Aopen+is%3Aissue+user%3Aprojectdiscovery+label%3AHacktoberfest
*/

View File

@ -0,0 +1,116 @@
package responsehighlighter
import (
"errors"
"fmt"
"regexp"
"strings"
"unicode"
"github.com/projectdiscovery/gologger"
)
// [0-9a-fA-F]{8} {2} - hexdump indexes (8 character hex value followed by two spaces)
// [0-9a-fA-F]{2} + - 2 character long hex values followed by one or two space (potentially wrapped with an ASCII color code, see below)
// \x1b\[(\d;?)+m - ASCII color code pattern
// \x1b\[0m - ASCII color code reset
// \|(.*)\|\n - ASCII representation of the input delimited by pipe characters
var hexDumpParsePattern = regexp.MustCompile(`([0-9a-fA-F]{8} {2})((?:(?:\x1b\[(?:\d;?)+m)?[0-9a-fA-F]{2}(?:\x1b\[0m)? +)+)\|(.*)\|\n`)
var hexValuePattern = regexp.MustCompile(`([a-fA-F0-9]{2})`)
type HighlightableHexDump struct {
index []string
hex []string
ascii []string
}
func NewHighlightableHexDump(rowSize int) HighlightableHexDump {
return HighlightableHexDump{index: make([]string, 0, rowSize), hex: make([]string, 0, rowSize), ascii: make([]string, 0, rowSize)}
}
func (hexDump HighlightableHexDump) len() int {
return len(hexDump.index)
}
func (hexDump HighlightableHexDump) String() string {
var result string
for i := 0; i < hexDump.len(); i++ {
result += hexDump.index[i] + hexDump.hex[i] + "|" + hexDump.ascii[i] + "|\n"
}
return result
}
func toHighLightedHexDump(hexDump, snippetToHighlight string) (HighlightableHexDump, error) {
hexDumpRowValues := hexDumpParsePattern.FindAllStringSubmatch(hexDump, -1)
if hexDumpRowValues == nil || len(hexDumpRowValues) != strings.Count(hexDump, "\n") {
message := "could not parse hexdump"
gologger.Warning().Msgf(message)
return HighlightableHexDump{}, errors.New(message)
}
result := NewHighlightableHexDump(len(hexDumpRowValues))
for _, currentHexDumpRowValues := range hexDumpRowValues {
result.index = append(result.index, currentHexDumpRowValues[1])
result.hex = append(result.hex, currentHexDumpRowValues[2])
result.ascii = append(result.ascii, currentHexDumpRowValues[3])
}
return result.highlight(snippetToHighlight), nil
}
func (hexDump HighlightableHexDump) highlight(snippetToColor string) HighlightableHexDump {
return highlightAsciiSection(highlightHexSection(hexDump, snippetToColor), snippetToColor)
}
func highlightHexSection(hexDump HighlightableHexDump, snippetToColor string) HighlightableHexDump {
var snippetHexCharactersMatchPattern string
for _, char := range snippetToColor {
snippetHexCharactersMatchPattern += fmt.Sprintf(`(%02x[ \n]+)`, char)
}
hexDump.hex = highlight(hexDump.hex, snippetHexCharactersMatchPattern, func(v string) string {
return hexValuePattern.ReplaceAllString(v, addColor("$1"))
})
return hexDump
}
func highlightAsciiSection(hexDump HighlightableHexDump, snippetToColor string) HighlightableHexDump {
var snippetCharactersMatchPattern string
for _, v := range snippetToColor {
snippetCharactersMatchPattern += fmt.Sprintf(`(%s\n*)`, regexp.QuoteMeta(string(v)))
}
hexDump.ascii = highlight(hexDump.ascii, snippetCharactersMatchPattern, func(v string) string {
if len(v) > 1 {
return addColor(string(v[0])) + v[1:] // do not color new line characters
}
return addColor(v)
})
return hexDump
}
func highlight(values []string, snippetCharactersMatchPattern string, replaceToFunc func(v string) string) []string {
rows := strings.Join(values, "\n")
compiledPattern := regexp.MustCompile(snippetCharactersMatchPattern)
for _, submatch := range compiledPattern.FindAllStringSubmatch(rows, -1) {
var replaceTo string
var replaceFrom string
for _, matchedValueWithSuffix := range submatch[1:] {
replaceFrom += matchedValueWithSuffix
replaceTo += replaceToFunc(matchedValueWithSuffix)
}
rows = strings.ReplaceAll(rows, replaceFrom, replaceTo)
}
return strings.Split(rows, "\n")
}
// IsASCII tests whether a string consists only of ASCII characters or not
func IsASCII(input string) bool {
for i := 0; i < len(input); i++ {
if input[i] > unicode.MaxASCII {
return false
}
}
return true
}

View File

@ -9,15 +9,22 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
)
var colorizer = aurora.NewAurora(true)
var colorFunction = aurora.Green
func Highlight(operatorResult *operators.Result, response string, noColor bool) string {
func Highlight(operatorResult *operators.Result, response string, noColor, hexDump bool) string {
result := response
if operatorResult != nil && !noColor {
for _, matches := range operatorResult.Matches {
if len(matches) > 0 {
for _, currentMatch := range matches {
result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String())
if hexDump {
highlightedHexDump, err := toHighLightedHexDump(result, currentMatch)
if err == nil {
result = highlightedHexDump.String()
}
} else {
result = strings.ReplaceAll(result, currentMatch, addColor(currentMatch))
}
}
}
}
@ -33,3 +40,7 @@ func CreateStatusCodeSnippet(response string, statusCode int) string {
}
return ""
}
func addColor(value string) string {
return colorFunction(value).String()
}

View File

@ -0,0 +1,78 @@
package responsehighlighter
import (
"encoding/hex"
"testing"
"github.com/stretchr/testify/assert"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
)
const input = "abcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmn"
func TestHexDumpHighlighting(t *testing.T) {
const highlightedHexDumpResponse = `00000000 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 |abcdefghijklmnab|
00000010 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 |cdefghijklmnabcd|
00000020 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 |efghijklmnabcdef|
00000030 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 |ghijklmnabcdefgh|
00000040 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a |ijklmnabcdefghij|
00000050 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c |klmnabcdefghijkl|
00000060 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |mnabcdefghijklmn|
00000070 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |abcdefghijklmn|
`
t.Run("Test highlighting when the snippet is wrapped", func(t *testing.T) {
result, err := toHighLightedHexDump(hex.Dump([]byte(input)), "defghij")
assert.Nil(t, err)
assert.Equal(t, highlightedHexDumpResponse, result.String())
})
t.Run("Test highlight when the snippet contains separator character", func(t *testing.T) {
value := "asdfasdfasda|basdfadsdfs|"
result, err := toHighLightedHexDump(hex.Dump([]byte(value)), "a|b")
expected := `00000000 61 73 64 66 61 73 64 66 61 73 64 61 7c 62 61 73 |asdfasdfasda|bas|
00000010 64 66 61 64 73 64 66 73 7c |dfadsdfs||
`
assert.Nil(t, err)
assert.Equal(t, expected, result.String())
})
}
func TestHighlight(t *testing.T) {
const multiSnippetHighlightHexDumpResponse = `00000000 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 |abcdefghijklmnab|
00000010 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 |cdefghijklmnabcd|
00000020 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 |efghijklmnabcdef|
00000030 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 |ghijklmnabcdefgh|
00000040 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a |ijklmnabcdefghij|
00000050 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c |klmnabcdefghijkl|
00000060 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |mnabcdefghijklmn|
00000070 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |abcdefghijklmn|
`
matches := map[string][]string{
"first": {"defghij"},
"second": {"ab"},
}
operatorResult := operators.Result{Matches: matches}
t.Run("Test highlighting when the snippet is wrapped", func(t *testing.T) {
result := Highlight(&operatorResult, hex.Dump([]byte(input)), false, true)
assert.Equal(t, multiSnippetHighlightHexDumpResponse, result)
})
t.Run("Test highlighting without hexdump", func(t *testing.T) {
result := Highlight(&operatorResult, input, false, false)
expected := `abcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmn`
assert.Equal(t, expected, result)
})
t.Run("Test the response is not modified if noColor is true", func(t *testing.T) {
result := Highlight(&operatorResult, input, true, false)
assert.Equal(t, input, result)
})
t.Run("Test the response is not modified if noColor is true", func(t *testing.T) {
result := Highlight(&operatorResult, hex.Dump([]byte(input)), true, true)
assert.Equal(t, hex.Dump([]byte(input)), result)
})
}

View File

@ -1,6 +1,7 @@
package dns
import (
"encoding/hex"
"net/url"
"github.com/pkg/errors"
@ -44,35 +45,45 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
}
// Send the request to the target servers
resp, err := request.dnsClient.Do(compiledRequest)
response, err := request.dnsClient.Do(compiledRequest)
if err != nil {
request.options.Output.Request(request.options.TemplateID, domain, "dns", err)
request.options.Progress.IncrementFailedRequestsBy(1)
}
if resp == nil {
if response == nil {
return errors.Wrap(err, "could not send dns request")
}
request.options.Progress.IncrementRequests()
request.options.Output.Request(request.options.TemplateID, domain, "dns", err)
gologger.Verbose().Msgf("[%s] Sent DNS request to %s", request.options.TemplateID, domain)
gologger.Verbose().Msgf("[%s] Sent DNS request to %s\n", request.options.TemplateID, domain)
outputEvent := request.responseToDSLMap(compiledRequest, resp, input, input)
outputEvent := request.responseToDSLMap(compiledRequest, response, input, input)
for k, v := range previous {
outputEvent[k] = v
}
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, resp.String(), request.options.Options.NoColor))
}
debug(event, request, domain, response.String())
callback(event)
return nil
}
func debug(event *output.InternalWrappedEvent, request *Request, domain string, response string) {
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped DNS response for %s\n", request.options.TemplateID, domain)
hexDump := false
if !responsehighlighter.IsASCII(response) {
hexDump = true
response = hex.Dump([]byte(response))
}
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump))
}
}
// isURL tests a string to determine if it is a well-structured url or not.
func isURL(toTest string) bool {
if _, err := url.ParseRequestURI(toTest); err != nil {

View File

@ -1,6 +1,8 @@
package file
import (
"encoding/hex"
"fmt"
"io/ioutil"
"os"
@ -49,20 +51,17 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
gologger.Error().Msgf("Could not read file path %s: %s\n", filePath, err)
return
}
dataStr := tostring.UnsafeToString(buffer)
fileContent := tostring.UnsafeToString(buffer)
gologger.Verbose().Msgf("[%s] Sent FILE request to %s", request.options.TemplateID, filePath)
outputEvent := request.responseToDSLMap(dataStr, input, filePath)
outputEvent := request.responseToDSLMap(fileContent, input, filePath)
for k, v := range previous {
outputEvent[k] = v
}
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, dataStr, request.options.Options.NoColor))
}
debug(event, request, filePath, fileContent)
callback(event)
}(data)
@ -76,3 +75,15 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
request.options.Progress.IncrementRequests()
return nil
}
func debug(event *output.InternalWrappedEvent, request *Request, filePath string, fileContent string) {
if request.options.Options.Debug || request.options.Options.DebugResponse {
hexDump := false
if !responsehighlighter.IsASCII(fileContent) {
hexDump = true
fileContent = hex.Dump([]byte(fileContent))
}
logHeader := fmt.Sprintf("[%s] Dumped file request for %s\n", request.options.TemplateID, filePath)
gologger.Debug().Msgf("%s\n%s", logHeader, responsehighlighter.Highlight(event.OperatorsResult, fileContent, request.options.Options.NoColor, hexDump))
}
}

View File

@ -17,42 +17,42 @@ import (
var _ protocols.Request = &Request{}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error {
func (request *Request) ExecuteWithResults(inputURL string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error {
instance, err := request.options.Browser.NewInstance()
if err != nil {
request.options.Output.Request(request.options.TemplateID, input, "headless", err)
request.options.Output.Request(request.options.TemplateID, inputURL, "headless", err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could get html element")
}
defer instance.Close()
parsed, err := url.Parse(input)
parsedURL, err := url.Parse(inputURL)
if err != nil {
request.options.Output.Request(request.options.TemplateID, input, "headless", err)
request.options.Output.Request(request.options.TemplateID, inputURL, "headless", err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could get html element")
}
out, page, err := instance.Run(parsed, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second)
out, page, err := instance.Run(parsedURL, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second)
if err != nil {
request.options.Output.Request(request.options.TemplateID, input, "headless", err)
request.options.Output.Request(request.options.TemplateID, inputURL, "headless", err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could get html element")
}
defer page.Close()
request.options.Output.Request(request.options.TemplateID, input, "headless", nil)
request.options.Output.Request(request.options.TemplateID, inputURL, "headless", nil)
request.options.Progress.IncrementRequests()
gologger.Verbose().Msgf("Sent Headless request to %s", input)
gologger.Verbose().Msgf("Sent Headless request to %s", inputURL)
reqBuilder := &strings.Builder{}
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, input)
gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, inputURL)
for _, act := range request.Steps {
reqBuilder.WriteString(act.String())
reqBuilder.WriteString("\n")
}
gologger.Print().Msgf("%s", reqBuilder.String())
gologger.Print().Msgf(reqBuilder.String())
}
var responseBody string
@ -60,18 +60,22 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
if err == nil {
responseBody, _ = html.HTML()
}
outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), input, input)
outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL)
for k, v := range out {
outputEvent[k] = v
}
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor))
}
debug(event, request, responseBody, inputURL)
callback(event)
return nil
}
func debug(event *output.InternalWrappedEvent, request *Request, responseBody string, input string) {
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n", request.options.TemplateID, input)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor, false))
}
}

View File

@ -2,6 +2,7 @@ package http
import (
"bytes"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
@ -450,7 +451,8 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
// Decode gbk response content-types
// gb18030 supersedes gb2312
if isContentTypeGbk(resp.Header.Get("Content-Type")) {
responseContentType := resp.Header.Get("Content-Type")
if isContentTypeGbk(responseContentType) {
dumpedResponse, err = decodegbk(dumpedResponse)
if err != nil {
return errors.Wrap(err, "could not gbk decode")
@ -509,10 +511,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
})
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, string(redirectedResponse), request.options.Options.NoColor))
}
debug(request, formedURL, redirectedResponse, responseContentType, event)
callback(event)
return nil
@ -532,3 +531,29 @@ func (request *Request) setCustomHeaders(req *generatedRequest) {
}
}
}
const CRLF = "\r\n"
func debug(request *Request, formedURL string, redirectedResponse []byte, responseContentType string, event *output.InternalWrappedEvent) {
if request.options.Options.Debug || request.options.Options.DebugResponse {
hexDump := false
response := string(redirectedResponse)
var headers string
if responseContentType == "" || responseContentType == "application/octet-stream" || (responseContentType == "application/x-www-form-urlencoded" && responsehighlighter.IsASCII(response)) {
hexDump = true
responseLines := strings.Split(response, CRLF)
for i, value := range responseLines {
headers += value + CRLF
if value == "" {
response = hex.Dump([]byte(strings.Join(responseLines[i+1:], "")))
break
}
}
}
logMessageHeader := fmt.Sprintf("[%s] Dumped HTTP response for %s\n", request.options.TemplateID, formedURL)
gologger.Debug().Msgf("%s\n%s", logMessageHeader, responsehighlighter.Highlight(event.OperatorsResult, headers, request.options.Options.NoColor, false))
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump))
}
}

View File

@ -190,8 +190,8 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
if request.options.Options.Debug || request.options.Options.DebugRequests {
requestOutput := reqBuilder.String()
gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", request.options.TemplateID, actualAddress)
gologger.Print().Msgf("%s\nHex: %s", requestOutput, hex.EncodeToString([]byte(requestOutput)))
gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n", request.options.TemplateID, actualAddress)
gologger.Print().Msgf("%s", hex.Dump([]byte(requestOutput)))
}
request.options.Output.Request(request.options.TemplateID, actualAddress, "network", err)
@ -274,14 +274,18 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
})
}
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped Network response for %s", request.options.TemplateID, actualAddress)
gologger.Print().Msgf("%s\nHex: %s", response, responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), request.options.Options.NoColor))
}
debug(event, request, response, actualAddress)
return nil
}
func debug(event *output.InternalWrappedEvent, request *Request, response string, actualAddress string) {
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped Network response for %s\n", request.options.TemplateID, actualAddress)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, hex.Dump([]byte(response)), request.options.Options.NoColor, true))
}
}
// getAddress returns the address of the host to make request to
func getAddress(toTest string) (string, error) {
if strings.Contains(toTest, "://") {