Added test cases for network protocol + misc

v2/pkg/protocols/dns/operators_test.go

@@ -114,6 +114,19 @@ func TestDNSOperatorMatch(t *testing.T) {
 		matched := request.Match(event, matcher)
 		require.True(t, matched, "could not match valid negative response matcher")
 	})
+
+	t.Run("invalid", func(t *testing.T) {
+		matcher := &matchers.Matcher{
+			Part:  "raw",
+			Type:  "word",
+			Words: []string{"random"},
+		}
+		err := matcher.CompileMatchers()
+		require.Nil(t, err, "could not compile matcher")
+
+		matched := request.Match(event, matcher)
+		require.False(t, matched, "could match invalid response matcher")
+	})
 }
 
 func TestDNSOperatorExtract(t *testing.T) {

v2/pkg/protocols/http/operators.go

@@ -2,7 +2,6 @@ package http
 
 import (
 	"net/http"
-	"net/http/httputil"
 	"strings"
 	"time"
 
@@ -86,11 +85,8 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r
 
 	data["host"] = host
 	data["matched"] = matched
-	if r.options.Options.JSONRequests {
-		data["request"] = rawReq
-		data["response"] = rawResp
-	}
-
+	data["request"] = rawReq
+	data["response"] = rawResp
 	data["content_length"] = resp.ContentLength
 	data["status_code"] = resp.StatusCode
 
@@ -103,11 +99,6 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r
 		data[k] = strings.Join(v, " ")
 	}
 	data["all_headers"] = headers
-
-	if r, err := httputil.DumpResponse(resp, true); err == nil {
-		rawString := string(r)
-		data["raw"] = rawString
-	}
 	data["duration"] = duration.Seconds()
 	data["template-id"] = r.options.TemplateID
 	data["template-info"] = r.options.TemplateInfo

v2/pkg/protocols/network/network_test.go

@@ -0,0 +1,36 @@
+package network
+
+import (
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNetworkCompileMake(t *testing.T) {
+	options := testutils.DefaultOptions
+
+	testutils.Init(options)
+	templateID := "testing-network"
+	request := &Request{
+		ID:       templateID,
+		Address:  []string{"{{Hostname}}", "{{Hostname}}:8082"},
+		ReadSize: 1024,
+		Inputs:   []*Input{&Input{Data: "test-data"}},
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: map[string]string{"severity": "low", "name": "test"},
+	})
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile network request")
+
+	require.Equal(t, 2, len(request.addresses), "could not get correct number of input address")
+	t.Run("check-host", func(t *testing.T) {
+		require.Equal(t, "{{Hostname}}", request.addresses[0].key, "could not get correct host")
+	})
+	t.Run("check-host-with-port", func(t *testing.T) {
+		require.Equal(t, "{{Hostname}}", request.addresses[1].key, "could not get correct host with port")
+		require.Equal(t, "8082", request.addresses[1].value, "could not get correct port for host")
+	})
+}

v2/pkg/protocols/network/operators.go

@@ -11,7 +11,7 @@ import (
 func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
 	partString := matcher.Part
 	switch partString {
-	case "body", "all", "":
+	case "body", "raw", "all", "":
 		partString = "data"
 	}
 
@@ -38,14 +38,9 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher)
 
 // Extract performs extracting operation for a extractor on model and returns true or false.
 func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
-	part, ok := data[extractor.Part]
-	if !ok {
-		return nil
-	}
-	partString := part.(string)
-
+	partString := extractor.Part
 	switch partString {
-	case "body", "all":
+	case "body", "raw", "all", "":
 		partString = "data"
 	}
 
@@ -66,14 +61,12 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext
 
 // responseToDSLMap converts a DNS response to a map for use in DSL matching
 func (r *Request) responseToDSLMap(req, resp string, host, matched string) output.InternalEvent {
-	data := make(output.InternalEvent, 4)
+	data := make(output.InternalEvent, 6)
 
 	// Some data regarding the request metadata
 	data["host"] = host
 	data["matched"] = matched
-	if r.options.Options.JSONRequests {
-		data["request"] = req
-	}
+	data["request"] = req
 	data["data"] = resp
 	data["template-id"] = r.options.TemplateID
 	data["template-info"] = r.options.TemplateInfo

v2/pkg/protocols/network/operators_test.go

@@ -0,0 +1,202 @@
+package network
+
+import (
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResponseToDSLMap(t *testing.T) {
+	options := testutils.DefaultOptions
+
+	testutils.Init(options)
+	templateID := "testing-network"
+	request := &Request{
+		ID:       templateID,
+		Address:  []string{"{{Hostname}}"},
+		ReadSize: 1024,
+		Inputs:   []*Input{&Input{Data: "test-data\r\n"}},
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: map[string]string{"severity": "low", "name": "test"},
+	})
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile network request")
+
+	req := "test-data\r\n"
+	resp := "resp-data\r\n"
+	event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
+	require.Len(t, event, 6, "could not get correct number of items in dsl map")
+	require.Equal(t, resp, event["data"], "could not get correct resp")
+}
+
+func TestNetworkOperatorMatch(t *testing.T) {
+	options := testutils.DefaultOptions
+
+	testutils.Init(options)
+	templateID := "testing-network"
+	request := &Request{
+		ID:       templateID,
+		Address:  []string{"{{Hostname}}"},
+		ReadSize: 1024,
+		Inputs:   []*Input{&Input{Data: "test-data\r\n"}},
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: map[string]string{"severity": "low", "name": "test"},
+	})
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile network request")
+
+	req := "test-data\r\n"
+	resp := "resp-data\r\nSTAT \r\n"
+	event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
+
+	t.Run("valid", func(t *testing.T) {
+		matcher := &matchers.Matcher{
+			Part:  "body",
+			Type:  "word",
+			Words: []string{"STAT "},
+		}
+		err = matcher.CompileMatchers()
+		require.Nil(t, err, "could not compile matcher")
+
+		matched := request.Match(event, matcher)
+		require.True(t, matched, "could not match valid response")
+	})
+
+	t.Run("negative", func(t *testing.T) {
+		matcher := &matchers.Matcher{
+			Part:     "raw",
+			Type:     "word",
+			Negative: true,
+			Words:    []string{"random"},
+		}
+		err := matcher.CompileMatchers()
+		require.Nil(t, err, "could not compile negative matcher")
+
+		matched := request.Match(event, matcher)
+		require.True(t, matched, "could not match valid negative response matcher")
+	})
+
+	t.Run("invalid", func(t *testing.T) {
+		matcher := &matchers.Matcher{
+			Part:  "raw",
+			Type:  "word",
+			Words: []string{"random"},
+		}
+		err := matcher.CompileMatchers()
+		require.Nil(t, err, "could not compile matcher")
+
+		matched := request.Match(event, matcher)
+		require.False(t, matched, "could match invalid response matcher")
+	})
+}
+
+func TestNetworkOperatorExtract(t *testing.T) {
+	options := testutils.DefaultOptions
+
+	testutils.Init(options)
+	templateID := "testing-network"
+	request := &Request{
+		ID:       templateID,
+		Address:  []string{"{{Hostname}}"},
+		ReadSize: 1024,
+		Inputs:   []*Input{&Input{Data: "test-data\r\n"}},
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: map[string]string{"severity": "low", "name": "test"},
+	})
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile network request")
+
+	req := "test-data\r\n"
+	resp := "resp-data\r\nSTAT \r\n1.1.1.1\r\n"
+	event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
+
+	t.Run("extract", func(t *testing.T) {
+		extractor := &extractors.Extractor{
+			Part:  "raw",
+			Type:  "regex",
+			Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
+		}
+		err = extractor.CompileExtractors()
+		require.Nil(t, err, "could not compile extractor")
+
+		data := request.Extract(event, extractor)
+		require.Greater(t, len(data), 0, "could not extractor valid response")
+		require.Equal(t, map[string]struct{}{"1.1.1.1": {}}, data, "could not extract correct data")
+	})
+
+	t.Run("kval", func(t *testing.T) {
+		extractor := &extractors.Extractor{
+			Type: "kval",
+			KVal: []string{"request"},
+		}
+		err = extractor.CompileExtractors()
+		require.Nil(t, err, "could not compile kval extractor")
+
+		data := request.Extract(event, extractor)
+		require.Greater(t, len(data), 0, "could not extractor kval valid response")
+		require.Equal(t, map[string]struct{}{req: {}}, data, "could not extract correct kval data")
+	})
+}
+
+func TestNetworkMakeResult(t *testing.T) {
+	options := testutils.DefaultOptions
+
+	testutils.Init(options)
+	templateID := "testing-network"
+	request := &Request{
+		ID:       templateID,
+		Address:  []string{"{{Hostname}}"},
+		ReadSize: 1024,
+		Inputs:   []*Input{&Input{Data: "test-data\r\n"}},
+		Operators: operators.Operators{
+			Matchers: []*matchers.Matcher{
+				&matchers.Matcher{
+					Name:  "test",
+					Part:  "raw",
+					Type:  "word",
+					Words: []string{"STAT "},
+				},
+			},
+			Extractors: []*extractors.Extractor{
+				&extractors.Extractor{
+					Part:  "raw",
+					Type:  "regex",
+					Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
+				},
+			},
+		},
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: map[string]string{"severity": "low", "name": "test"},
+	})
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile network request")
+
+	req := "test-data\r\n"
+	resp := "resp-data\rSTAT \r\n1.1.1.1\n"
+	event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
+	finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
+	event["ip"] = "192.168.1.1"
+	if request.CompiledOperators != nil {
+		result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract)
+		if ok && result != nil {
+			finalEvent.OperatorsResult = result
+			finalEvent.Results = request.MakeResultEvent(finalEvent)
+		}
+	}
+	require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
+	require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
+	require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
+}

v2/pkg/protocols/network/request_test.go

@@ -0,0 +1,101 @@
+package network
+
+import (
+	"encoding/hex"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDNSExecuteWithResults(t *testing.T) {
+	options := testutils.DefaultOptions
+
+	testutils.Init(options)
+	templateID := "testing-network"
+	request := &Request{
+		ID:       templateID,
+		Address:  []string{"{{Hostname}}:80"},
+		ReadSize: 2048,
+		Inputs:   []*Input{&Input{Data: "GET / HTTP/1.1\r\n\r\n"}},
+		Operators: operators.Operators{
+			Matchers: []*matchers.Matcher{
+				&matchers.Matcher{
+					Name:  "test",
+					Part:  "raw",
+					Type:  "word",
+					Words: []string{"400 - Bad Request"},
+				},
+			},
+			Extractors: []*extractors.Extractor{
+				&extractors.Extractor{
+					Part:  "raw",
+					Type:  "regex",
+					Regex: []string{"<h1>.*</h1>"},
+				},
+			},
+		},
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: map[string]string{"severity": "low", "name": "test"},
+	})
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile network request")
+
+	var finalEvent *output.InternalWrappedEvent
+	t.Run("domain-valid", func(t *testing.T) {
+		metadata := make(output.InternalEvent)
+		previous := make(output.InternalEvent)
+		err := request.ExecuteWithResults("example.com", metadata, previous, func(event *output.InternalWrappedEvent) {
+			finalEvent = event
+		})
+		require.Nil(t, err, "could not execute network request")
+	})
+	require.NotNil(t, finalEvent, "could not get event output from request")
+	require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
+	require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
+	require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
+	require.Equal(t, "<h1>400 - Bad Request</h1>", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
+	finalEvent = nil
+
+	t.Run("invalid-port-override", func(t *testing.T) {
+		metadata := make(output.InternalEvent)
+		previous := make(output.InternalEvent)
+		err := request.ExecuteWithResults("example.com:11211", metadata, previous, func(event *output.InternalWrappedEvent) {
+			finalEvent = event
+		})
+		require.Nil(t, err, "could not execute network request")
+	})
+	require.NotNil(t, finalEvent, "could not get event output from request")
+	require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
+	require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
+	require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
+	require.Equal(t, "<h1>400 - Bad Request</h1>", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
+	finalEvent = nil
+
+	request.Inputs[0].Type = "hex"
+	request.Inputs[0].Data = hex.EncodeToString([]byte("GET / HTTP/1.1\r\n\r\n"))
+
+	t.Run("hex-to-string", func(t *testing.T) {
+		metadata := make(output.InternalEvent)
+		previous := make(output.InternalEvent)
+		err := request.ExecuteWithResults("example.com", metadata, previous, func(event *output.InternalWrappedEvent) {
+			finalEvent = event
+		})
+		require.Nil(t, err, "could not execute network request")
+	})
+	require.NotNil(t, finalEvent, "could not get event output from request")
+	require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
+	require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
+	require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
+	require.Equal(t, "<h1>400 - Bad Request</h1>", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
+	finalEvent = nil
+
+	request.Inputs[0].Type = ""
+	request.Inputs[0].Data = "GET / HTTP/1.1\r\n\r\n"
+}