[feature] Add coloring to debug information #999

* extracted common logic and made sure that all requests implement the same interface

v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go

@@ -0,0 +1,26 @@
+package eventcreator
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+)
+
+func CreateEvent(request protocols.Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent {
+	return CreateEventWithAdditionalOptions(request, outputEvent, func(internalWrappedEvent *output.InternalWrappedEvent) {})
+}
+
+func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent output.InternalEvent, addAdditionalOptions func(internalWrappedEvent *output.InternalWrappedEvent)) *output.InternalWrappedEvent {
+	event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
+	for _, compiledOperator := range request.GetCompiledOperators() {
+		if compiledOperator != nil {
+			result, ok := compiledOperator.Execute(outputEvent, request.Match, request.Extract)
+			if ok && result != nil {
+				event.OperatorsResult = result
+				addAdditionalOptions(event)
+				event.Results = request.MakeResultEvent(event)
+			}
+		}
+	}
+
+	return event
+}

v2/pkg/protocols/dns/dns.go

@@ -74,6 +74,10 @@ type Request struct {
 	Recursion bool `yaml:"recursion,omitempty" jsonschema:"title=recurse all servers,description=Recursion determines if resolver should recurse all records to get fresh results"`
 }
 
+func (request *Request) GetCompiledOperators() []*operators.Operators {
+	return []*operators.Operators{request.CompiledOperators}
+}
+
 // GetID returns the unique ID of the request if any.
 func (request *Request) GetID() string {
 	return request.ID

v2/pkg/protocols/dns/operators.go

@@ -27,7 +27,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
 	}
 
 	switch matcher.GetType() {
-	case matchers.StatusMatcher: // TODO is this correct?
+	case matchers.StatusMatcher:
 		return matcher.Result(matcher.MatchStatusCode(item.(int))), []string{}
 	case matchers.SizeMatcher:
 		return matcher.Result(matcher.MatchSize(len(types.ToString(item)))), []string{}

v2/pkg/protocols/dns/request.go

@@ -8,6 +8,7 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
 )
 
@@ -55,7 +56,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
 		outputEvent[k] = v
 	}
 
-	event := createEvent(request, outputEvent)
+	event := eventcreator.CreateEvent(request, outputEvent)
 
 	if request.options.Options.Debug || request.options.Options.DebugResponse {
 		gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain)
@@ -66,20 +67,6 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
 	return nil
 }
 
-func createEvent(request *Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent {
-	event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
-
-	if request.CompiledOperators != nil {
-		result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract)
-		if ok && result != nil {
-			event.OperatorsResult = result
-			event.Results = request.MakeResultEvent(event)
-		}
-	}
-
-	return event
-}
-
 // isURL tests a string to determine if it is a well-structured url or not.
 func isURL(toTest string) bool {
 	if _, err := url.ParseRequestURI(toTest); err != nil {

v2/pkg/protocols/file/operators.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
@@ -133,6 +134,10 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []
 	return results
 }
 
+func (request *Request) GetCompiledOperators() []*operators.Operators {
+	return []*operators.Operators{request.CompiledOperators}
+}
+
 func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),

v2/pkg/protocols/file/request.go

@@ -10,6 +10,7 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
 )
@@ -56,7 +57,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
 				outputEvent[k] = v
 			}
 
-			event := createEvent(request, outputEvent)
+			event := eventcreator.CreateEvent(request, outputEvent)
 
 			if request.options.Options.Debug || request.options.Options.DebugResponse {
 				gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath)
@@ -75,17 +76,3 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
 	request.options.Progress.IncrementRequests()
 	return nil
 }
-
-func createEvent(request *Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent {
-	event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
-
-	if request.CompiledOperators != nil {
-		result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract)
-		if ok && result != nil {
-			event.OperatorsResult = result
-			event.Results = request.MakeResultEvent(event)
-		}
-	}
-
-	return event
-}

v2/pkg/protocols/headless/operators.go

@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
@@ -105,6 +106,10 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []
 	return results
 }
 
+func (request *Request) GetCompiledOperators() []*operators.Operators {
+	return []*operators.Operators{request.CompiledOperators}
+}
+
 func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),

v2/pkg/protocols/headless/request.go

@@ -10,6 +10,7 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
 )
 
@@ -64,7 +65,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 		outputEvent[k] = v
 	}
 
-	event := createEvent(request, outputEvent)
+	event := eventcreator.CreateEvent(request, outputEvent)
 
 	if request.options.Options.Debug || request.options.Options.DebugResponse {
 		gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input)
@@ -74,17 +75,3 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 	callback(event)
 	return nil
 }
-
-func createEvent(request *Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent {
-	event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
-
-	if request.CompiledOperators != nil {
-		result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract)
-		if ok && result != nil {
-			event.OperatorsResult = result
-			event.Results = request.MakeResultEvent(event)
-		}
-	}
-
-	return event
-}

v2/pkg/protocols/http/operators.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
@@ -143,6 +144,10 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []
 	return results
 }
 
+func (request *Request) GetCompiledOperators() []*operators.Operators {
+	return []*operators.Operators{request.CompiledOperators}
+}
+
 func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),

v2/pkg/protocols/http/request.go

@@ -20,6 +20,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
@@ -263,7 +264,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
 const drainReqSize = int64(8 * 1024)
 
 // executeRequest executes the actual generated request and returns error if occurred
-func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previous output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error {
+func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error {
 	request.setCustomHeaders(generatedRequest)
 
 	var (
@@ -447,7 +448,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 	}
 	outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
 	outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse)
-	for k, v := range previous {
+	for k, v := range previousEvent {
 		finalEvent[k] = v
 	}
 	for k, v := range outputEvent {
@@ -457,12 +458,14 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 	if request.ReqCondition {
 		for k, v := range outputEvent {
 			key := fmt.Sprintf("%s_%d", k, requestCount)
-			previous[key] = v
+			previousEvent[key] = v
 			finalEvent[key] = v
 		}
 	}
 
-	event := createEvent(request, outputEvent, finalEvent, generatedRequest)
+	event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, func(internalWrappedEvent *output.InternalWrappedEvent) {
+		internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
+	})
 
 	if request.options.Options.Debug || request.options.Options.DebugResponse {
 		gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL)
@@ -473,21 +476,6 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 	return nil
 }
 
-func createEvent(request *Request, outputEvent output.InternalEvent, finalEvent output.InternalEvent, generatedRequest *generatedRequest) *output.InternalWrappedEvent {
-	event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
-
-	if request.CompiledOperators != nil {
-		result, ok := request.CompiledOperators.Execute(finalEvent, request.Match, request.Extract)
-		if ok && result != nil {
-			event.OperatorsResult = result
-			event.OperatorsResult.PayloadValues = generatedRequest.meta
-			event.Results = request.MakeResultEvent(event)
-		}
-	}
-
-	return event
-}
-
 // setCustomHeaders sets the custom headers for generated request
 func (request *Request) setCustomHeaders(req *generatedRequest) {
 	for k, v := range request.customHeaders {

v2/pkg/protocols/network/operators.go

@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
@@ -106,6 +107,10 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []
 	return results
 }
 
+func (request *Request) GetCompiledOperators() []*operators.Operators {
+	return []*operators.Operators{request.CompiledOperators}
+}
+
 func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),

v2/pkg/protocols/network/request.go

@@ -16,6 +16,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
@@ -202,11 +203,14 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
 		outputEvent[k] = v
 	}
 
-	event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
+	var event *output.InternalWrappedEvent
 	if interactURL == "" {
-		event := createEvent(request, outputEvent, event, payloads)
+		event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, func(wrappedEvent *output.InternalWrappedEvent) {
+			wrappedEvent.OperatorsResult.PayloadValues = payloads
+		})
 		callback(event)
 	} else if request.options.Interactsh != nil {
+		event = &output.InternalWrappedEvent{InternalEvent: outputEvent}
 		request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{
 			MakeResultFunc: request.MakeResultEvent,
 			Event:          event,
@@ -224,19 +228,6 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
 	return nil
 }
 
-func createEvent(request *Request, outputEvent output.InternalEvent, event *output.InternalWrappedEvent, payloads map[string]interface{}) *output.InternalWrappedEvent {
-	if request.CompiledOperators != nil {
-		result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract)
-		if ok && result != nil {
-			event.OperatorsResult = result
-			event.OperatorsResult.PayloadValues = payloads
-			event.Results = request.MakeResultEvent(event)
-		}
-	}
-
-	return event
-}
-
 // getAddress returns the address of the host to make request to
 func getAddress(toTest string) (string, error) {
 	if strings.Contains(toTest, "://") {

v2/pkg/protocols/offlinehttp/operators.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
@@ -135,6 +136,10 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []
 	return results
 }
 
+func (request *Request) GetCompiledOperators() []*operators.Operators {
+	return request.compiledOperators
+}
+
 func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),

v2/pkg/protocols/offlinehttp/request.go

@@ -11,9 +11,9 @@ import (
 	"github.com/remeh/sizedwaitgroup"
 
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
 )
 
@@ -85,10 +85,8 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
 				outputEvent[k] = v
 			}
 
-			for _, operator := range request.compiledOperators {
+			event := eventcreator.CreateEvent(request, outputEvent)
-				event := createEvent(outputEvent, operator, request)
+			callback(event)
-				callback(event)
-			}
 		}(data)
 	})
 	wg.Wait()
@@ -101,20 +99,6 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
 	return nil
 }
 
-func createEvent(outputEvent map[string]interface{}, operator *operators.Operators, request *Request) *output.InternalWrappedEvent {
-	event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
-
-	if request.compiledOperators != nil {
-		result, ok := operator.Execute(outputEvent, request.Match, request.Extract)
-		if ok && event.OperatorsResult != nil {
-			event.OperatorsResult = result
-			event.Results = request.MakeResultEvent(event)
-		}
-	}
-
-	return event
-}
-
 // headersToString converts http headers to string
 func headersToString(headers http.Header) string {
 	builder := &strings.Builder{}

v2/pkg/protocols/protocols.go

@@ -82,6 +82,10 @@ type Request interface {
 	Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
 	// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 	ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error
+	// MakeResultEvent creates a result event from internal wrapped event
+	MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent
+	// GetCompiledOperators returns a list of the compiled operators
+	GetCompiledOperators() []*operators.Operators
 }
 
 // OutputEventCallback is a callback event for any results found during scanning.