mirror of https://github.com/daffainfo/nuclei.git
feat: In case of binary data, show a hexadecimal view as well #1080
parent
e0afa2cee4
commit
04e3c0165a
|
@ -175,10 +175,3 @@ func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags
|
|||
currentFlag.Group(groupName)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
HacktoberFest update: Below, you can find our ticket recommendations. Tasks with the "good first issue" label are suitable for first time contributors. If you have other ideas, or need help with getting started, join our Discord channel or reach out to @forgedhallpass.
|
||||
|
||||
https://github.com/issues?q=is%3Aopen+is%3Aissue+user%3Aprojectdiscovery+label%3AHacktoberfest
|
||||
|
||||
*/
|
||||
|
|
|
@ -0,0 +1,116 @@
|
|||
package responsehighlighter
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
)
|
||||
|
||||
// [0-9a-fA-F]{8} {2} - hexdump indexes (8 character hex value followed by two spaces)
|
||||
// [0-9a-fA-F]{2} + - 2 character long hex values followed by one or two space (potentially wrapped with an ASCII color code, see below)
|
||||
// \x1b\[(\d;?)+m - ASCII color code pattern
|
||||
// \x1b\[0m - ASCII color code reset
|
||||
// \|(.*)\|\n - ASCII representation of the input delimited by pipe characters
|
||||
var hexDumpParsePattern = regexp.MustCompile(`([0-9a-fA-F]{8} {2})((?:(?:\x1b\[(?:\d;?)+m)?[0-9a-fA-F]{2}(?:\x1b\[0m)? +)+)\|(.*)\|\n`)
|
||||
var hexValuePattern = regexp.MustCompile(`([a-fA-F0-9]{2})`)
|
||||
|
||||
type HighlightableHexDump struct {
|
||||
index []string
|
||||
hex []string
|
||||
ascii []string
|
||||
}
|
||||
|
||||
func NewHighlightableHexDump(rowSize int) HighlightableHexDump {
|
||||
return HighlightableHexDump{index: make([]string, 0, rowSize), hex: make([]string, 0, rowSize), ascii: make([]string, 0, rowSize)}
|
||||
}
|
||||
|
||||
func (hexDump HighlightableHexDump) len() int {
|
||||
return len(hexDump.index)
|
||||
}
|
||||
|
||||
func (hexDump HighlightableHexDump) String() string {
|
||||
var result string
|
||||
for i := 0; i < hexDump.len(); i++ {
|
||||
result += hexDump.index[i] + hexDump.hex[i] + "|" + hexDump.ascii[i] + "|\n"
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func toHighLightedHexDump(hexDump, snippetToHighlight string) (HighlightableHexDump, error) {
|
||||
hexDumpRowValues := hexDumpParsePattern.FindAllStringSubmatch(hexDump, -1)
|
||||
if hexDumpRowValues == nil || len(hexDumpRowValues) != strings.Count(hexDump, "\n") {
|
||||
message := "could not parse hexdump"
|
||||
gologger.Warning().Msgf(message)
|
||||
return HighlightableHexDump{}, errors.New(message)
|
||||
}
|
||||
|
||||
result := NewHighlightableHexDump(len(hexDumpRowValues))
|
||||
for _, currentHexDumpRowValues := range hexDumpRowValues {
|
||||
result.index = append(result.index, currentHexDumpRowValues[1])
|
||||
result.hex = append(result.hex, currentHexDumpRowValues[2])
|
||||
result.ascii = append(result.ascii, currentHexDumpRowValues[3])
|
||||
}
|
||||
return result.highlight(snippetToHighlight), nil
|
||||
}
|
||||
|
||||
func (hexDump HighlightableHexDump) highlight(snippetToColor string) HighlightableHexDump {
|
||||
return highlightAsciiSection(highlightHexSection(hexDump, snippetToColor), snippetToColor)
|
||||
}
|
||||
|
||||
func highlightHexSection(hexDump HighlightableHexDump, snippetToColor string) HighlightableHexDump {
|
||||
var snippetHexCharactersMatchPattern string
|
||||
for _, char := range snippetToColor {
|
||||
snippetHexCharactersMatchPattern += fmt.Sprintf(`(%02x[ \n]+)`, char)
|
||||
}
|
||||
|
||||
hexDump.hex = highlight(hexDump.hex, snippetHexCharactersMatchPattern, func(v string) string {
|
||||
return hexValuePattern.ReplaceAllString(v, addColor("$1"))
|
||||
})
|
||||
|
||||
return hexDump
|
||||
}
|
||||
|
||||
func highlightAsciiSection(hexDump HighlightableHexDump, snippetToColor string) HighlightableHexDump {
|
||||
var snippetCharactersMatchPattern string
|
||||
for _, v := range snippetToColor {
|
||||
snippetCharactersMatchPattern += fmt.Sprintf(`(%s\n*)`, regexp.QuoteMeta(string(v)))
|
||||
}
|
||||
|
||||
hexDump.ascii = highlight(hexDump.ascii, snippetCharactersMatchPattern, func(v string) string {
|
||||
if len(v) > 1 {
|
||||
return addColor(string(v[0])) + v[1:] // do not color new line characters
|
||||
}
|
||||
return addColor(v)
|
||||
})
|
||||
|
||||
return hexDump
|
||||
}
|
||||
|
||||
func highlight(values []string, snippetCharactersMatchPattern string, replaceToFunc func(v string) string) []string {
|
||||
rows := strings.Join(values, "\n")
|
||||
compiledPattern := regexp.MustCompile(snippetCharactersMatchPattern)
|
||||
for _, submatch := range compiledPattern.FindAllStringSubmatch(rows, -1) {
|
||||
var replaceTo string
|
||||
var replaceFrom string
|
||||
for _, matchedValueWithSuffix := range submatch[1:] {
|
||||
replaceFrom += matchedValueWithSuffix
|
||||
replaceTo += replaceToFunc(matchedValueWithSuffix)
|
||||
}
|
||||
rows = strings.ReplaceAll(rows, replaceFrom, replaceTo)
|
||||
}
|
||||
return strings.Split(rows, "\n")
|
||||
}
|
||||
|
||||
// IsASCII tests whether a string consists only of ASCII characters or not
|
||||
func IsASCII(input string) bool {
|
||||
for i := 0; i < len(input); i++ {
|
||||
if input[i] > unicode.MaxASCII {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
|
@ -9,15 +9,22 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
|
||||
)
|
||||
|
||||
var colorizer = aurora.NewAurora(true)
|
||||
var colorFunction = aurora.Green
|
||||
|
||||
func Highlight(operatorResult *operators.Result, response string, noColor bool) string {
|
||||
func Highlight(operatorResult *operators.Result, response string, noColor, hexDump bool) string {
|
||||
result := response
|
||||
if operatorResult != nil && !noColor {
|
||||
for _, matches := range operatorResult.Matches {
|
||||
if len(matches) > 0 {
|
||||
for _, currentMatch := range matches {
|
||||
result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String())
|
||||
if hexDump {
|
||||
highlightedHexDump, err := toHighLightedHexDump(result, currentMatch)
|
||||
if err == nil {
|
||||
result = highlightedHexDump.String()
|
||||
}
|
||||
} else {
|
||||
result = strings.ReplaceAll(result, currentMatch, addColor(currentMatch))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -33,3 +40,7 @@ func CreateStatusCodeSnippet(response string, statusCode int) string {
|
|||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func addColor(value string) string {
|
||||
return colorFunction(value).String()
|
||||
}
|
||||
|
|
|
@ -0,0 +1,78 @@
|
|||
package responsehighlighter
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
|
||||
)
|
||||
|
||||
const input = "abcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmn"
|
||||
|
||||
func TestHexDumpHighlighting(t *testing.T) {
|
||||
const highlightedHexDumpResponse = `00000000 61 62 63 [32m64[0m [32m65[0m [32m66[0m [32m67[0m [32m68[0m [32m69[0m [32m6a[0m 6b 6c 6d 6e 61 62 |abc[32md[0m[32me[0m[32mf[0m[32mg[0m[32mh[0m[32mi[0m[32mj[0mklmnab|
|
||||
00000010 63 [32m64[0m [32m65[0m [32m66[0m [32m67[0m [32m68[0m [32m69[0m [32m6a[0m 6b 6c 6d 6e 61 62 63 [32m64[0m |c[32md[0m[32me[0m[32mf[0m[32mg[0m[32mh[0m[32mi[0m[32mj[0mklmnabc[32md[0m|
|
||||
00000020 [32m65[0m [32m66[0m [32m67[0m [32m68[0m [32m69[0m [32m6a[0m 6b 6c 6d 6e 61 62 63 [32m64[0m [32m65[0m [32m66[0m |[32me[0m[32mf[0m[32mg[0m[32mh[0m[32mi[0m[32mj[0mklmnabc[32md[0m[32me[0m[32mf[0m|
|
||||
00000030 [32m67[0m [32m68[0m [32m69[0m [32m6a[0m 6b 6c 6d 6e 61 62 63 [32m64[0m [32m65[0m [32m66[0m [32m67[0m [32m68[0m |[32mg[0m[32mh[0m[32mi[0m[32mj[0mklmnabc[32md[0m[32me[0m[32mf[0m[32mg[0m[32mh[0m|
|
||||
00000040 [32m69[0m [32m6a[0m 6b 6c 6d 6e 61 62 63 [32m64[0m [32m65[0m [32m66[0m [32m67[0m [32m68[0m [32m69[0m [32m6a[0m |[32mi[0m[32mj[0mklmnabc[32md[0m[32me[0m[32mf[0m[32mg[0m[32mh[0m[32mi[0m[32mj[0m|
|
||||
00000050 6b 6c 6d 6e 61 62 63 [32m64[0m [32m65[0m [32m66[0m [32m67[0m [32m68[0m [32m69[0m [32m6a[0m 6b 6c |klmnabc[32md[0m[32me[0m[32mf[0m[32mg[0m[32mh[0m[32mi[0m[32mj[0mkl|
|
||||
00000060 6d 6e 61 62 63 [32m64[0m [32m65[0m [32m66[0m [32m67[0m [32m68[0m [32m69[0m [32m6a[0m 6b 6c 6d 6e |mnabc[32md[0m[32me[0m[32mf[0m[32mg[0m[32mh[0m[32mi[0m[32mj[0mklmn|
|
||||
00000070 61 62 63 [32m64[0m [32m65[0m [32m66[0m [32m67[0m [32m68[0m [32m69[0m [32m6a[0m 6b 6c 6d 6e |abc[32md[0m[32me[0m[32mf[0m[32mg[0m[32mh[0m[32mi[0m[32mj[0mklmn|
|
||||
`
|
||||
t.Run("Test highlighting when the snippet is wrapped", func(t *testing.T) {
|
||||
result, err := toHighLightedHexDump(hex.Dump([]byte(input)), "defghij")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, highlightedHexDumpResponse, result.String())
|
||||
})
|
||||
|
||||
t.Run("Test highlight when the snippet contains separator character", func(t *testing.T) {
|
||||
value := "asdfasdfasda|basdfadsdfs|"
|
||||
result, err := toHighLightedHexDump(hex.Dump([]byte(value)), "a|b")
|
||||
|
||||
expected := `00000000 61 73 64 66 61 73 64 66 61 73 64 [32m61[0m [32m7c[0m [32m62[0m 61 73 |asdfasdfasd[32ma[0m[32m|[0m[32mb[0mas|
|
||||
00000010 64 66 61 64 73 64 66 73 7c |dfadsdfs||
|
||||
`
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, expected, result.String())
|
||||
})
|
||||
}
|
||||
|
||||
func TestHighlight(t *testing.T) {
|
||||
const multiSnippetHighlightHexDumpResponse = `00000000 [32m61[0m [32m62[0m 63 [32m64[0m [32m65[0m [32m66[0m [32m67[0m [32m68[0m [32m69[0m [32m6a[0m 6b 6c 6d 6e [32m61[0m [32m62[0m |[32ma[0m[32mb[0mc[32md[0m[32me[0m[32mf[0m[32mg[0m[32mh[0m[32mi[0m[32mj[0mklmn[32ma[0m[32mb[0m|
|
||||
00000010 63 [32m64[0m [32m65[0m [32m66[0m [32m67[0m [32m68[0m [32m69[0m [32m6a[0m 6b 6c 6d 6e [32m61[0m [32m62[0m 63 [32m64[0m |c[32md[0m[32me[0m[32mf[0m[32mg[0m[32mh[0m[32mi[0m[32mj[0mklmn[32ma[0m[32mb[0mc[32md[0m|
|
||||
00000020 [32m65[0m [32m66[0m [32m67[0m [32m68[0m [32m69[0m [32m6a[0m 6b 6c 6d 6e [32m61[0m [32m62[0m 63 [32m64[0m [32m65[0m [32m66[0m |[32me[0m[32mf[0m[32mg[0m[32mh[0m[32mi[0m[32mj[0mklmn[32ma[0m[32mb[0mc[32md[0m[32me[0m[32mf[0m|
|
||||
00000030 [32m67[0m [32m68[0m [32m69[0m [32m6a[0m 6b 6c 6d 6e [32m61[0m [32m62[0m 63 [32m64[0m [32m65[0m [32m66[0m [32m67[0m [32m68[0m |[32mg[0m[32mh[0m[32mi[0m[32mj[0mklmn[32ma[0m[32mb[0mc[32md[0m[32me[0m[32mf[0m[32mg[0m[32mh[0m|
|
||||
00000040 [32m69[0m [32m6a[0m 6b 6c 6d 6e [32m61[0m [32m62[0m 63 [32m64[0m [32m65[0m [32m66[0m [32m67[0m [32m68[0m [32m69[0m [32m6a[0m |[32mi[0m[32mj[0mklmn[32ma[0m[32mb[0mc[32md[0m[32me[0m[32mf[0m[32mg[0m[32mh[0m[32mi[0m[32mj[0m|
|
||||
00000050 6b 6c 6d 6e [32m61[0m [32m62[0m 63 [32m64[0m [32m65[0m [32m66[0m [32m67[0m [32m68[0m [32m69[0m [32m6a[0m 6b 6c |klmn[32ma[0m[32mb[0mc[32md[0m[32me[0m[32mf[0m[32mg[0m[32mh[0m[32mi[0m[32mj[0mkl|
|
||||
00000060 6d 6e [32m61[0m [32m62[0m 63 [32m64[0m [32m65[0m [32m66[0m [32m67[0m [32m68[0m [32m69[0m [32m6a[0m 6b 6c 6d 6e |mn[32ma[0m[32mb[0mc[32md[0m[32me[0m[32mf[0m[32mg[0m[32mh[0m[32mi[0m[32mj[0mklmn|
|
||||
00000070 [32m61[0m [32m62[0m 63 [32m64[0m [32m65[0m [32m66[0m [32m67[0m [32m68[0m [32m69[0m [32m6a[0m 6b 6c 6d 6e |[32ma[0m[32mb[0mc[32md[0m[32me[0m[32mf[0m[32mg[0m[32mh[0m[32mi[0m[32mj[0mklmn|
|
||||
`
|
||||
matches := map[string][]string{
|
||||
"first": {"defghij"},
|
||||
"second": {"ab"},
|
||||
}
|
||||
operatorResult := operators.Result{Matches: matches}
|
||||
|
||||
t.Run("Test highlighting when the snippet is wrapped", func(t *testing.T) {
|
||||
result := Highlight(&operatorResult, hex.Dump([]byte(input)), false, true)
|
||||
assert.Equal(t, multiSnippetHighlightHexDumpResponse, result)
|
||||
})
|
||||
|
||||
t.Run("Test highlighting without hexdump", func(t *testing.T) {
|
||||
result := Highlight(&operatorResult, input, false, false)
|
||||
expected := `[32mab[0mc[32mdefghij[0mklmn[32mab[0mc[32mdefghij[0mklmn[32mab[0mc[32mdefghij[0mklmn[32mab[0mc[32mdefghij[0mklmn[32mab[0mc[32mdefghij[0mklmn[32mab[0mc[32mdefghij[0mklmn[32mab[0mc[32mdefghij[0mklmn[32mab[0mc[32mdefghij[0mklmn[32mab[0mc[32mdefghij[0mklmn`
|
||||
assert.Equal(t, expected, result)
|
||||
})
|
||||
|
||||
t.Run("Test the response is not modified if noColor is true", func(t *testing.T) {
|
||||
result := Highlight(&operatorResult, input, true, false)
|
||||
assert.Equal(t, input, result)
|
||||
})
|
||||
|
||||
t.Run("Test the response is not modified if noColor is true", func(t *testing.T) {
|
||||
result := Highlight(&operatorResult, hex.Dump([]byte(input)), true, true)
|
||||
assert.Equal(t, hex.Dump([]byte(input)), result)
|
||||
})
|
||||
}
|
|
@ -1,6 +1,7 @@
|
|||
package dns
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"net/url"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
@ -44,35 +45,45 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
|
|||
}
|
||||
|
||||
// Send the request to the target servers
|
||||
resp, err := request.dnsClient.Do(compiledRequest)
|
||||
response, err := request.dnsClient.Do(compiledRequest)
|
||||
if err != nil {
|
||||
request.options.Output.Request(request.options.TemplateID, domain, "dns", err)
|
||||
request.options.Progress.IncrementFailedRequestsBy(1)
|
||||
}
|
||||
if resp == nil {
|
||||
if response == nil {
|
||||
return errors.Wrap(err, "could not send dns request")
|
||||
}
|
||||
request.options.Progress.IncrementRequests()
|
||||
|
||||
request.options.Output.Request(request.options.TemplateID, domain, "dns", err)
|
||||
gologger.Verbose().Msgf("[%s] Sent DNS request to %s", request.options.TemplateID, domain)
|
||||
gologger.Verbose().Msgf("[%s] Sent DNS request to %s\n", request.options.TemplateID, domain)
|
||||
|
||||
outputEvent := request.responseToDSLMap(compiledRequest, resp, input, input)
|
||||
outputEvent := request.responseToDSLMap(compiledRequest, response, input, input)
|
||||
for k, v := range previous {
|
||||
outputEvent[k] = v
|
||||
}
|
||||
|
||||
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
|
||||
|
||||
if request.options.Options.Debug || request.options.Options.DebugResponse {
|
||||
gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain)
|
||||
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, resp.String(), request.options.Options.NoColor))
|
||||
}
|
||||
debug(event, request, domain, response.String())
|
||||
|
||||
callback(event)
|
||||
return nil
|
||||
}
|
||||
|
||||
func debug(event *output.InternalWrappedEvent, request *Request, domain string, response string) {
|
||||
if request.options.Options.Debug || request.options.Options.DebugResponse {
|
||||
gologger.Debug().Msgf("[%s] Dumped DNS response for %s\n", request.options.TemplateID, domain)
|
||||
|
||||
hexDump := false
|
||||
if !responsehighlighter.IsASCII(response) {
|
||||
hexDump = true
|
||||
response = hex.Dump([]byte(response))
|
||||
}
|
||||
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump))
|
||||
}
|
||||
}
|
||||
|
||||
// isURL tests a string to determine if it is a well-structured url or not.
|
||||
func isURL(toTest string) bool {
|
||||
if _, err := url.ParseRequestURI(toTest); err != nil {
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
package file
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
||||
|
@ -49,20 +51,17 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
|
|||
gologger.Error().Msgf("Could not read file path %s: %s\n", filePath, err)
|
||||
return
|
||||
}
|
||||
dataStr := tostring.UnsafeToString(buffer)
|
||||
fileContent := tostring.UnsafeToString(buffer)
|
||||
|
||||
gologger.Verbose().Msgf("[%s] Sent FILE request to %s", request.options.TemplateID, filePath)
|
||||
outputEvent := request.responseToDSLMap(dataStr, input, filePath)
|
||||
outputEvent := request.responseToDSLMap(fileContent, input, filePath)
|
||||
for k, v := range previous {
|
||||
outputEvent[k] = v
|
||||
}
|
||||
|
||||
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
|
||||
|
||||
if request.options.Options.Debug || request.options.Options.DebugResponse {
|
||||
gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath)
|
||||
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, dataStr, request.options.Options.NoColor))
|
||||
}
|
||||
debug(event, request, filePath, fileContent)
|
||||
|
||||
callback(event)
|
||||
}(data)
|
||||
|
@ -76,3 +75,15 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
|
|||
request.options.Progress.IncrementRequests()
|
||||
return nil
|
||||
}
|
||||
|
||||
func debug(event *output.InternalWrappedEvent, request *Request, filePath string, fileContent string) {
|
||||
if request.options.Options.Debug || request.options.Options.DebugResponse {
|
||||
hexDump := false
|
||||
if !responsehighlighter.IsASCII(fileContent) {
|
||||
hexDump = true
|
||||
fileContent = hex.Dump([]byte(fileContent))
|
||||
}
|
||||
logHeader := fmt.Sprintf("[%s] Dumped file request for %s\n", request.options.TemplateID, filePath)
|
||||
gologger.Debug().Msgf("%s\n%s", logHeader, responsehighlighter.Highlight(event.OperatorsResult, fileContent, request.options.Options.NoColor, hexDump))
|
||||
}
|
||||
}
|
||||
|
|
|
@ -17,42 +17,42 @@ import (
|
|||
var _ protocols.Request = &Request{}
|
||||
|
||||
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
|
||||
func (request *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error {
|
||||
func (request *Request) ExecuteWithResults(inputURL string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error {
|
||||
instance, err := request.options.Browser.NewInstance()
|
||||
if err != nil {
|
||||
request.options.Output.Request(request.options.TemplateID, input, "headless", err)
|
||||
request.options.Output.Request(request.options.TemplateID, inputURL, "headless", err)
|
||||
request.options.Progress.IncrementFailedRequestsBy(1)
|
||||
return errors.Wrap(err, "could get html element")
|
||||
}
|
||||
defer instance.Close()
|
||||
|
||||
parsed, err := url.Parse(input)
|
||||
parsedURL, err := url.Parse(inputURL)
|
||||
if err != nil {
|
||||
request.options.Output.Request(request.options.TemplateID, input, "headless", err)
|
||||
request.options.Output.Request(request.options.TemplateID, inputURL, "headless", err)
|
||||
request.options.Progress.IncrementFailedRequestsBy(1)
|
||||
return errors.Wrap(err, "could get html element")
|
||||
}
|
||||
out, page, err := instance.Run(parsed, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second)
|
||||
out, page, err := instance.Run(parsedURL, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second)
|
||||
if err != nil {
|
||||
request.options.Output.Request(request.options.TemplateID, input, "headless", err)
|
||||
request.options.Output.Request(request.options.TemplateID, inputURL, "headless", err)
|
||||
request.options.Progress.IncrementFailedRequestsBy(1)
|
||||
return errors.Wrap(err, "could get html element")
|
||||
}
|
||||
defer page.Close()
|
||||
|
||||
request.options.Output.Request(request.options.TemplateID, input, "headless", nil)
|
||||
request.options.Output.Request(request.options.TemplateID, inputURL, "headless", nil)
|
||||
request.options.Progress.IncrementRequests()
|
||||
gologger.Verbose().Msgf("Sent Headless request to %s", input)
|
||||
gologger.Verbose().Msgf("Sent Headless request to %s", inputURL)
|
||||
|
||||
reqBuilder := &strings.Builder{}
|
||||
if request.options.Options.Debug || request.options.Options.DebugRequests {
|
||||
gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, input)
|
||||
gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, inputURL)
|
||||
|
||||
for _, act := range request.Steps {
|
||||
reqBuilder.WriteString(act.String())
|
||||
reqBuilder.WriteString("\n")
|
||||
}
|
||||
gologger.Print().Msgf("%s", reqBuilder.String())
|
||||
gologger.Print().Msgf(reqBuilder.String())
|
||||
}
|
||||
|
||||
var responseBody string
|
||||
|
@ -60,18 +60,22 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
|
|||
if err == nil {
|
||||
responseBody, _ = html.HTML()
|
||||
}
|
||||
outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), input, input)
|
||||
outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL)
|
||||
for k, v := range out {
|
||||
outputEvent[k] = v
|
||||
}
|
||||
|
||||
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
|
||||
|
||||
if request.options.Options.Debug || request.options.Options.DebugResponse {
|
||||
gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input)
|
||||
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor))
|
||||
}
|
||||
debug(event, request, responseBody, inputURL)
|
||||
|
||||
callback(event)
|
||||
return nil
|
||||
}
|
||||
|
||||
func debug(event *output.InternalWrappedEvent, request *Request, responseBody string, input string) {
|
||||
if request.options.Options.Debug || request.options.Options.DebugResponse {
|
||||
gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n", request.options.TemplateID, input)
|
||||
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor, false))
|
||||
}
|
||||
}
|
||||
|
|
|
@ -2,6 +2,7 @@ package http
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
|
@ -450,7 +451,8 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
|
|||
|
||||
// Decode gbk response content-types
|
||||
// gb18030 supersedes gb2312
|
||||
if isContentTypeGbk(resp.Header.Get("Content-Type")) {
|
||||
responseContentType := resp.Header.Get("Content-Type")
|
||||
if isContentTypeGbk(responseContentType) {
|
||||
dumpedResponse, err = decodegbk(dumpedResponse)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not gbk decode")
|
||||
|
@ -509,10 +511,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
|
|||
internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
|
||||
})
|
||||
|
||||
if request.options.Options.Debug || request.options.Options.DebugResponse {
|
||||
gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL)
|
||||
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, string(redirectedResponse), request.options.Options.NoColor))
|
||||
}
|
||||
debug(request, formedURL, redirectedResponse, responseContentType, event)
|
||||
|
||||
callback(event)
|
||||
return nil
|
||||
|
@ -532,3 +531,29 @@ func (request *Request) setCustomHeaders(req *generatedRequest) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
const CRLF = "\r\n"
|
||||
|
||||
func debug(request *Request, formedURL string, redirectedResponse []byte, responseContentType string, event *output.InternalWrappedEvent) {
|
||||
if request.options.Options.Debug || request.options.Options.DebugResponse {
|
||||
hexDump := false
|
||||
response := string(redirectedResponse)
|
||||
|
||||
var headers string
|
||||
if responseContentType == "" || responseContentType == "application/octet-stream" || (responseContentType == "application/x-www-form-urlencoded" && responsehighlighter.IsASCII(response)) {
|
||||
hexDump = true
|
||||
responseLines := strings.Split(response, CRLF)
|
||||
for i, value := range responseLines {
|
||||
headers += value + CRLF
|
||||
if value == "" {
|
||||
response = hex.Dump([]byte(strings.Join(responseLines[i+1:], "")))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
logMessageHeader := fmt.Sprintf("[%s] Dumped HTTP response for %s\n", request.options.TemplateID, formedURL)
|
||||
|
||||
gologger.Debug().Msgf("%s\n%s", logMessageHeader, responsehighlighter.Highlight(event.OperatorsResult, headers, request.options.Options.NoColor, false))
|
||||
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump))
|
||||
}
|
||||
}
|
||||
|
|
|
@ -190,8 +190,8 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
|
|||
|
||||
if request.options.Options.Debug || request.options.Options.DebugRequests {
|
||||
requestOutput := reqBuilder.String()
|
||||
gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", request.options.TemplateID, actualAddress)
|
||||
gologger.Print().Msgf("%s\nHex: %s", requestOutput, hex.EncodeToString([]byte(requestOutput)))
|
||||
gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n", request.options.TemplateID, actualAddress)
|
||||
gologger.Print().Msgf("%s", hex.Dump([]byte(requestOutput)))
|
||||
}
|
||||
|
||||
request.options.Output.Request(request.options.TemplateID, actualAddress, "network", err)
|
||||
|
@ -274,14 +274,18 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
|
|||
})
|
||||
}
|
||||
|
||||
if request.options.Options.Debug || request.options.Options.DebugResponse {
|
||||
gologger.Debug().Msgf("[%s] Dumped Network response for %s", request.options.TemplateID, actualAddress)
|
||||
gologger.Print().Msgf("%s\nHex: %s", response, responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), request.options.Options.NoColor))
|
||||
}
|
||||
debug(event, request, response, actualAddress)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func debug(event *output.InternalWrappedEvent, request *Request, response string, actualAddress string) {
|
||||
if request.options.Options.Debug || request.options.Options.DebugResponse {
|
||||
gologger.Debug().Msgf("[%s] Dumped Network response for %s\n", request.options.TemplateID, actualAddress)
|
||||
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, hex.Dump([]byte(response)), request.options.Options.NoColor, true))
|
||||
}
|
||||
}
|
||||
|
||||
// getAddress returns the address of the host to make request to
|
||||
func getAddress(toTest string) (string, error) {
|
||||
if strings.Contains(toTest, "://") {
|
||||
|
|
Loading…
Reference in New Issue