daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[feature] Add coloring to debug information #999 

* extracted common logic and made sure that all requests implement the same interface
dev
forgedhallpass 2021-10-01 16:52:38 +03:00
parent b8a2b2a1c4
commit 08cd7a4ba7
15 changed files with 82 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go Normal file
View File

@ -0,0 +1,26 @@
package eventcreator
import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
)
func CreateEvent(request protocols.Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent {
return CreateEventWithAdditionalOptions(request, outputEvent, func(internalWrappedEvent *output.InternalWrappedEvent) {})
}
func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent output.InternalEvent, addAdditionalOptions func(internalWrappedEvent *output.InternalWrappedEvent)) *output.InternalWrappedEvent {
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
for _, compiledOperator := range request.GetCompiledOperators() {
if compiledOperator != nil {
result, ok := compiledOperator.Execute(outputEvent, request.Match, request.Extract)
if ok && result != nil {
event.OperatorsResult = result
addAdditionalOptions(event)
event.Results = request.MakeResultEvent(event)
}
}
}
return event
}

4
v2/pkg/protocols/dns/dns.go
View File

@ -74,6 +74,10 @@ type Request struct {
Recursion bool `yaml:"recursion,omitempty" jsonschema:"title=recurse all servers,description=Recursion determines if resolver should recurse all records to get fresh results"`
}
func (request *Request) GetCompiledOperators() []*operators.Operators {
return []*operators.Operators{request.CompiledOperators}
}
// GetID returns the unique ID of the request if any.
func (request *Request) GetID() string {
return request.ID

2
v2/pkg/protocols/dns/operators.go
View File

@ -27,7 +27,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
}
switch matcher.GetType() {
case matchers.StatusMatcher: // TODO is this correct?
case matchers.StatusMatcher:
return matcher.Result(matcher.MatchStatusCode(item.(int))), []string{}
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(types.ToString(item)))), []string{}

17
v2/pkg/protocols/dns/request.go
View File

@ -8,6 +8,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
)
@ -55,7 +56,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
outputEvent[k] = v
}
event := createEvent(request, outputEvent)
event := eventcreator.CreateEvent(request, outputEvent)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain)
@ -66,20 +67,6 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
return nil
}
func createEvent(request *Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent {
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.Results = request.MakeResultEvent(event)
}
}
return event
}
// isURL tests a string to determine if it is a well-structured url or not.
func isURL(toTest string) bool {
if _, err := url.ParseRequestURI(toTest); err != nil {

5
v2/pkg/protocols/file/operators.go
View File

@ -6,6 +6,7 @@ import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
@ -133,6 +134,10 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []
return results
}
func (request *Request) GetCompiledOperators() []*operators.Operators {
return []*operators.Operators{request.CompiledOperators}
}
func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),

17
v2/pkg/protocols/file/request.go
View File

@ -10,6 +10,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
)
@ -56,7 +57,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
outputEvent[k] = v
}
event := createEvent(request, outputEvent)
event := eventcreator.CreateEvent(request, outputEvent)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath)
@ -75,17 +76,3 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
request.options.Progress.IncrementRequests()
return nil
}
func createEvent(request *Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent {
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.Results = request.MakeResultEvent(event)
}
}
return event
}

5
v2/pkg/protocols/headless/operators.go
View File

@ -4,6 +4,7 @@ import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
@ -105,6 +106,10 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []
return results
}
func (request *Request) GetCompiledOperators() []*operators.Operators {
return []*operators.Operators{request.CompiledOperators}
}
func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),

17
v2/pkg/protocols/headless/request.go
View File

@ -10,6 +10,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
)
@ -64,7 +65,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
outputEvent[k] = v
}
event := createEvent(request, outputEvent)
event := eventcreator.CreateEvent(request, outputEvent)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input)
@ -74,17 +75,3 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
callback(event)
return nil
}
func createEvent(request *Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent {
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.Results = request.MakeResultEvent(event)
}
}
return event
}

5
v2/pkg/protocols/http/operators.go
View File

@ -7,6 +7,7 @@ import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
@ -143,6 +144,10 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []
return results
}
func (request *Request) GetCompiledOperators() []*operators.Operators {
return []*operators.Operators{request.CompiledOperators}
}
func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),

26
v2/pkg/protocols/http/request.go
View File

@ -20,6 +20,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
@ -263,7 +264,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
const drainReqSize = int64(8 * 1024)
// executeRequest executes the actual generated request and returns error if occurred
func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previous output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error {
func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error {
request.setCustomHeaders(generatedRequest)
var (
@ -447,7 +448,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
}
outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse)
for k, v := range previous {
for k, v := range previousEvent {
finalEvent[k] = v
}
for k, v := range outputEvent {
@ -457,12 +458,14 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
if request.ReqCondition {
for k, v := range outputEvent {
key := fmt.Sprintf("%s_%d", k, requestCount)
previous[key] = v
previousEvent[key] = v
finalEvent[key] = v
}
}
event := createEvent(request, outputEvent, finalEvent, generatedRequest)
event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, func(internalWrappedEvent *output.InternalWrappedEvent) {
internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
})
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL)
@ -473,21 +476,6 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
return nil
}
func createEvent(request *Request, outputEvent output.InternalEvent, finalEvent output.InternalEvent, generatedRequest *generatedRequest) *output.InternalWrappedEvent {
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(finalEvent, request.Match, request.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.OperatorsResult.PayloadValues = generatedRequest.meta
event.Results = request.MakeResultEvent(event)
}
}
return event
}
// setCustomHeaders sets the custom headers for generated request
func (request *Request) setCustomHeaders(req *generatedRequest) {
for k, v := range request.customHeaders {

5
v2/pkg/protocols/network/operators.go
View File

@ -4,6 +4,7 @@ import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
@ -106,6 +107,10 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []
return results
}
func (request *Request) GetCompiledOperators() []*operators.Operators {
return []*operators.Operators{request.CompiledOperators}
}
func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),

21
v2/pkg/protocols/network/request.go
View File

@ -16,6 +16,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
@ -202,11 +203,14 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
outputEvent[k] = v
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
var event *output.InternalWrappedEvent
if interactURL == "" {
event := createEvent(request, outputEvent, event, payloads)
event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, func(wrappedEvent *output.InternalWrappedEvent) {
wrappedEvent.OperatorsResult.PayloadValues = payloads
})
callback(event)
} else if request.options.Interactsh != nil {
event = &output.InternalWrappedEvent{InternalEvent: outputEvent}
request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
Event: event,
@ -224,19 +228,6 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
return nil
}
func createEvent(request *Request, outputEvent output.InternalEvent, event *output.InternalWrappedEvent, payloads map[string]interface{}) *output.InternalWrappedEvent {
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.OperatorsResult.PayloadValues = payloads
event.Results = request.MakeResultEvent(event)
}
}
return event
}
// getAddress returns the address of the host to make request to
func getAddress(toTest string) (string, error) {
if strings.Contains(toTest, "://") {

5
v2/pkg/protocols/offlinehttp/operators.go
View File

@ -7,6 +7,7 @@ import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
@ -135,6 +136,10 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []
return results
}
func (request *Request) GetCompiledOperators() []*operators.Operators {
return request.compiledOperators
}
func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),

22
v2/pkg/protocols/offlinehttp/request.go
View File

@ -11,9 +11,9 @@ import (
"github.com/remeh/sizedwaitgroup"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
)
@ -85,10 +85,8 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
outputEvent[k] = v
}
for _, operator := range request.compiledOperators {
event := createEvent(outputEvent, operator, request)
callback(event)
}
event := eventcreator.CreateEvent(request, outputEvent)
callback(event)
}(data)
})
wg.Wait()
@ -101,20 +99,6 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
return nil
}
func createEvent(outputEvent map[string]interface{}, operator *operators.Operators, request *Request) *output.InternalWrappedEvent {
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if request.compiledOperators != nil {
result, ok := operator.Execute(outputEvent, request.Match, request.Extract)
if ok && event.OperatorsResult != nil {
event.OperatorsResult = result
event.Results = request.MakeResultEvent(event)
}
}
return event
}
// headersToString converts http headers to string
func headersToString(headers http.Header) string {
builder := &strings.Builder{}

4
v2/pkg/protocols/protocols.go
View File

@ -82,6 +82,10 @@ type Request interface {
Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error
// MakeResultEvent creates a result event from internal wrapped event
MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent
// GetCompiledOperators returns a list of the compiled operators
GetCompiledOperators() []*operators.Operators
}
// OutputEventCallback is a callback event for any results found during scanning.