daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into more-protocols

dev
Sandeep Singh 2021-11-09 18:27:13 +05:30 committed by GitHub
parent 5e6b6c6b14 be3e8eecfb
commit f78ff42dbe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 129 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -1,4 +1,4 @@
FROM golang:1.17.2-alpine as build-env
FROM golang:1.17.3-alpine as build-env
RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest
FROM alpine:3.14

2
integration_tests/http/raw-unsafe-request.yaml
View File

@ -7,7 +7,7 @@ info:
requests:
- raw:
- |
- |+
GET / HTTP/1.1
Host:
Content-Length: 4

2
v2/go.mod
View File

@ -64,6 +64,8 @@ require (
moul.io/http2curl v1.0.0
)
require github.com/weppos/publicsuffix-go v0.15.0
require (
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect
github.com/PuerkitoBio/goquery v1.6.0 // indirect

16
v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go
View File

@ -77,7 +77,13 @@ func highlightHexSection(hexDump HighlightableHexDump, snippetToColor string) Hi
func highlightAsciiSection(hexDump HighlightableHexDump, snippetToColor string) HighlightableHexDump {
var snippetCharactersMatchPattern string
for _, v := range snippetToColor {
snippetCharactersMatchPattern += fmt.Sprintf(`(%s\n*)`, regexp.QuoteMeta(string(v)))
var value string
if IsASCIIPrintable(v) {
value = regexp.QuoteMeta(string(v))
} else {
value = "."
}
snippetCharactersMatchPattern += fmt.Sprintf(`(%s\n*)`, value)
}
hexDump.ascii = highlight(hexDump.ascii, snippetCharactersMatchPattern, func(v string) string {
@ -105,6 +111,10 @@ func highlight(values []string, snippetCharactersMatchPattern string, replaceToF
return strings.Split(rows, "\n")
}
func HasBinaryContent(input string) bool {
return !IsASCII(input)
}
// IsASCII tests whether a string consists only of ASCII characters or not
func IsASCII(input string) bool {
for i := 0; i < len(input); i++ {
@ -114,3 +124,7 @@ func IsASCII(input string) bool {
}
return true
}
func IsASCIIPrintable(input rune) bool {
return input > 32 && input < unicode.MaxASCII
}

14
v2/pkg/protocols/dns/request.go
View File

@ -71,22 +71,22 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
debug(event, request, domain, response.String())
dumpResponse(event, request.options, response.String(), domain)
callback(event)
return nil
}
func debug(event *output.InternalWrappedEvent, request *Request, domain string, response string) {
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped DNS response for %s\n", request.options.TemplateID, domain)
func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response string, domain string) {
cliOptions := requestOptions.Options
if cliOptions.Debug || cliOptions.DebugResponse {
hexDump := false
if !responsehighlighter.IsASCII(response) {
if responsehighlighter.HasBinaryContent(response) {
hexDump = true
response = hex.Dump([]byte(response))
}
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump))
highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, response, cliOptions.NoColor, hexDump)
gologger.Debug().Msgf("[%s] Dumped DNS response for %s\n\n%s", requestOptions.TemplateID, domain, highlightedResponse)
}
}

14
v2/pkg/protocols/file/request.go
View File

@ -2,7 +2,6 @@ package file
import (
"encoding/hex"
"fmt"
"io/ioutil"
"os"
@ -67,7 +66,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
debug(event, request, filePath, fileContent)
dumpResponse(event, request.options, fileContent, filePath)
callback(event)
}(data)
@ -82,14 +81,15 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
return nil
}
func debug(event *output.InternalWrappedEvent, request *Request, filePath string, fileContent string) {
if request.options.Options.Debug || request.options.Options.DebugResponse {
func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, fileContent string, filePath string) {
cliOptions := requestOptions.Options
if cliOptions.Debug || cliOptions.DebugResponse {
hexDump := false
if !responsehighlighter.IsASCII(fileContent) {
if responsehighlighter.HasBinaryContent(fileContent) {
hexDump = true
fileContent = hex.Dump([]byte(fileContent))
}
logHeader := fmt.Sprintf("[%s] Dumped file request for %s\n", request.options.TemplateID, filePath)
gologger.Debug().Msgf("%s\n%s", logHeader, responsehighlighter.Highlight(event.OperatorsResult, fileContent, request.options.Options.NoColor, hexDump))
highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, fileContent, cliOptions.NoColor, hexDump)
gologger.Debug().Msgf("[%s] Dumped file request for %s\n\n%s", requestOptions.TemplateID, filePath, highlightedResponse)
}
}

11
v2/pkg/protocols/headless/request.go
View File

@ -73,15 +73,16 @@ func (request *Request) ExecuteWithResults(inputURL string, metadata, previous o
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
debug(event, request, responseBody, inputURL)
dumpResponse(event, request.options, responseBody, inputURL)
callback(event)
return nil
}
func debug(event *output.InternalWrappedEvent, request *Request, responseBody string, input string) {
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n", request.options.TemplateID, input)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor, false))
func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, responseBody string, input string) {
cliOptions := requestOptions.Options
if cliOptions.Debug || cliOptions.DebugResponse {
highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, responseBody, cliOptions.NoColor, false)
gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n\n%s", requestOptions.TemplateID, input, highlightedResponse)
}
}

3
v2/pkg/protocols/http/build_request.go
View File

@ -227,6 +227,9 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest
// Unsafe option uses rawhttp library
if r.request.Unsafe {
if len(r.options.Options.CustomHeaders) > 0 {
_ = rawRequestData.TryFillCustomHeaders(r.options.Options.CustomHeaders)
}
unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: generatorValues, original: r.request}
return unsafeReq, nil
}

27
v2/pkg/protocols/http/raw/raw.go
View File

@ -3,6 +3,7 @@ package raw
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
@ -146,3 +147,29 @@ func fixUnsafeRequestPath(baseURL *url.URL, requestPath string, request []byte)
fixed := bytes.Replace(request, []byte(requestPath), []byte(fixedPath), 1)
return fixed
}
// TryFillCustomHeaders after the Host header
func (r *Request) TryFillCustomHeaders(headers []string) error {
unsafeBytes := bytes.ToLower(r.UnsafeRawBytes)
// locate first host header
hostHeaderIndex := bytes.Index(unsafeBytes, []byte("host:"))
if hostHeaderIndex > 0 {
// attempt to locate next newline
newLineIndex := bytes.Index(unsafeBytes[hostHeaderIndex:], []byte("\r\n"))
if newLineIndex > 0 {
newLineIndex += hostHeaderIndex + 2
// insert custom headers
var buf bytes.Buffer
buf.Write(r.UnsafeRawBytes[:newLineIndex])
for _, header := range headers {
buf.WriteString(fmt.Sprintf("%s\r\n", header))
}
buf.Write(r.UnsafeRawBytes[newLineIndex:])
r.UnsafeRawBytes = buf.Bytes()
return nil
}
return errors.New("no new line found at the end of host header")
}
return errors.New("no host header found")
}

10
v2/pkg/protocols/http/raw/raw_test.go
View File

@ -75,3 +75,13 @@ Connection: close`, "https://test.com/test/", true)
require.Nil(t, err, "could not parse unsafe request")
require.Contains(t, string(request.UnsafeRawBytes), "GET /test/manager/html", "Could not parse unsafe method request path correctly")
}
func TestTryFillCustomHeaders(t *testing.T) {
testValue := "GET /manager/html HTTP/1.1\r\nHost: Test\r\n"
expected := "GET /test/manager/html HTTP/1.1\r\nHost: Test\r\ntest: test\r\n"
request, err := Parse(testValue, "https://test.com/test/", true)
require.Nil(t, err, "could not parse unsafe request")
err = request.TryFillCustomHeaders([]string{"test: test"})
require.Nil(t, err, "could not add custom headers")
require.Equal(t, expected, string(request.UnsafeRawBytes), "actual value and expected value are different")
}

43
v2/pkg/protocols/http/request.go
View File

@ -513,7 +513,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
})
debug(request, formedURL, redirectedResponse, responseContentType, event)
dumpResponse(event, request.options, redirectedResponse, formedURL, responseContentType)
callback(event)
return nil
@ -536,26 +536,33 @@ func (request *Request) setCustomHeaders(req *generatedRequest) {
const CRLF = "\r\n"
func debug(request *Request, formedURL string, redirectedResponse []byte, responseContentType string, event *output.InternalWrappedEvent) {
if request.options.Options.Debug || request.options.Options.DebugResponse {
hexDump := false
func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, redirectedResponse []byte, formedURL string, responseContentType string) {
cliOptions := requestOptions.Options
if cliOptions.Debug || cliOptions.DebugResponse {
response := string(redirectedResponse)
var headers string
if responseContentType == "" || responseContentType == "application/octet-stream" || (responseContentType == "application/x-www-form-urlencoded" && responsehighlighter.IsASCII(response)) {
hexDump = true
responseLines := strings.Split(response, CRLF)
for i, value := range responseLines {
headers += value + CRLF
if value == "" {
response = hex.Dump([]byte(strings.Join(responseLines[i+1:], "")))
break
}
}
var highlightedResult string
if responseContentType == "application/octet-stream" || ((responseContentType == "" || responseContentType == "application/x-www-form-urlencoded") && responsehighlighter.HasBinaryContent(response)) {
highlightedResult = createResponseHexDump(event, response, cliOptions.NoColor)
} else {
highlightedResult = responsehighlighter.Highlight(event.OperatorsResult, response, cliOptions.NoColor, false)
}
logMessageHeader := fmt.Sprintf("[%s] Dumped HTTP response for %s\n", request.options.TemplateID, formedURL)
gologger.Debug().Msgf("%s\n%s", logMessageHeader, responsehighlighter.Highlight(event.OperatorsResult, headers, request.options.Options.NoColor, false))
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump))
gologger.Debug().Msgf("[%s] Dumped HTTP response for %s\n\n%s", requestOptions.TemplateID, formedURL, highlightedResult)
}
}
func createResponseHexDump(event *output.InternalWrappedEvent, response string, noColor bool) string {
CRLFs := CRLF + CRLF
headerEndIndex := strings.Index(response, CRLFs) + len(CRLFs)
if headerEndIndex > 0 {
headers := response[0:headerEndIndex]
responseBodyHexDump := hex.Dump([]byte(response[headerEndIndex:]))
highlightedHeaders := responsehighlighter.Highlight(event.OperatorsResult, headers, noColor, false)
highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, responseBodyHexDump, noColor, true)
return fmt.Sprintf("%s\n%s", highlightedHeaders, highlightedResponse)
} else {
return responsehighlighter.Highlight(event.OperatorsResult, hex.Dump([]byte(response)), noColor, true)
}
}

41
v2/pkg/protocols/network/request.go
View File

@ -13,6 +13,7 @@ import (
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
@ -189,9 +190,8 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
request.options.Progress.IncrementRequests()
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n", request.options.TemplateID, actualAddress)
requestBytes := []byte(reqBuilder.String())
gologger.Print().Msgf("%s", hex.Dump(requestBytes))
gologger.Debug().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n%s", request.options.TemplateID, actualAddress, hex.Dump(requestBytes))
if request.options.Options.VerboseVerbose {
gologger.Print().Msgf("\nCompact HEX view:\n%s", hex.EncodeToString(requestBytes))
}
@ -277,29 +277,38 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
})
}
debug(event, request, response, actualAddress)
dumpResponse(event, request.options, response, actualAddress)
return nil
}
func debug(event *output.InternalWrappedEvent, request *Request, response string, actualAddress string) {
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped Network response for %s\n", request.options.TemplateID, actualAddress)
func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response string, actualAddress string) {
cliOptions := requestOptions.Options
if cliOptions.Debug || cliOptions.DebugResponse {
requestBytes := []byte(response)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, hex.Dump(requestBytes), request.options.Options.NoColor, true))
if request.options.Options.VerboseVerbose {
var allMatches []string
for _, namedMatch := range event.OperatorsResult.Matches {
for _, matchElement := range namedMatch {
allMatches = append(allMatches, hex.EncodeToString([]byte(matchElement)))
}
}
event.OperatorsResult.Matches["compact"] = allMatches
gologger.Print().Msgf("\nCompact HEX view:\n%s", responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), request.options.Options.NoColor, false))
highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, hex.Dump(requestBytes), cliOptions.NoColor, true)
gologger.Debug().Msgf("[%s] Dumped Network response for %s\n\n%s", requestOptions.TemplateID, actualAddress, highlightedResponse)
if cliOptions.VerboseVerbose {
displayCompactHexView(event, response, cliOptions.NoColor)
}
}
}
func displayCompactHexView(event *output.InternalWrappedEvent, response string, noColor bool) {
operatorsResult := event.OperatorsResult
if operatorsResult != nil {
var allMatches []string
for _, namedMatch := range operatorsResult.Matches {
for _, matchElement := range namedMatch {
allMatches = append(allMatches, hex.EncodeToString([]byte(matchElement)))
}
}
tempOperatorResult := &operators.Result{Matches: map[string][]string{"matchesInHex": allMatches}}
gologger.Print().Msgf("\nCompact HEX view:\n%s", responsehighlighter.Highlight(tempOperatorResult, hex.EncodeToString([]byte(response)), noColor, false))
}
}
// getAddress returns the address of the host to make request to
func getAddress(toTest string) (string, error) {
if strings.Contains(toTest, "://") {