From 493b90589710b79e6829f6b4f0229b9f9884f56c Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 3 Feb 2021 13:07:24 +0530 Subject: [PATCH] Added test cases for network protocol + misc --- v2/pkg/protocols/dns/operators_test.go | 13 ++ v2/pkg/protocols/http/operators.go | 13 +- v2/pkg/protocols/network/network_test.go | 36 ++++ v2/pkg/protocols/network/operators.go | 17 +- v2/pkg/protocols/network/operators_test.go | 202 +++++++++++++++++++++ v2/pkg/protocols/network/request_test.go | 101 +++++++++++ 6 files changed, 359 insertions(+), 23 deletions(-) create mode 100644 v2/pkg/protocols/network/network_test.go create mode 100644 v2/pkg/protocols/network/operators_test.go create mode 100644 v2/pkg/protocols/network/request_test.go diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index af9a712e..0118edd1 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -114,6 +114,19 @@ func TestDNSOperatorMatch(t *testing.T) { matched := request.Match(event, matcher) require.True(t, matched, "could not match valid negative response matcher") }) + + t.Run("invalid", func(t *testing.T) { + matcher := &matchers.Matcher{ + Part: "raw", + Type: "word", + Words: []string{"random"}, + } + err := matcher.CompileMatchers() + require.Nil(t, err, "could not compile matcher") + + matched := request.Match(event, matcher) + require.False(t, matched, "could match invalid response matcher") + }) } func TestDNSOperatorExtract(t *testing.T) { diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 5b59a772..988f7220 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -2,7 +2,6 @@ package http import ( "net/http" - "net/http/httputil" "strings" "time" @@ -86,11 +85,8 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r data["host"] = host data["matched"] = matched - if r.options.Options.JSONRequests { - data["request"] = rawReq - data["response"] = rawResp - } - + data["request"] = rawReq + data["response"] = rawResp data["content_length"] = resp.ContentLength data["status_code"] = resp.StatusCode @@ -103,11 +99,6 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r data[k] = strings.Join(v, " ") } data["all_headers"] = headers - - if r, err := httputil.DumpResponse(resp, true); err == nil { - rawString := string(r) - data["raw"] = rawString - } data["duration"] = duration.Seconds() data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo diff --git a/v2/pkg/protocols/network/network_test.go b/v2/pkg/protocols/network/network_test.go new file mode 100644 index 00000000..d3c3cb76 --- /dev/null +++ b/v2/pkg/protocols/network/network_test.go @@ -0,0 +1,36 @@ +package network + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/stretchr/testify/require" +) + +func TestNetworkCompileMake(t *testing.T) { + options := testutils.DefaultOptions + + testutils.Init(options) + templateID := "testing-network" + request := &Request{ + ID: templateID, + Address: []string{"{{Hostname}}", "{{Hostname}}:8082"}, + ReadSize: 1024, + Inputs: []*Input{&Input{Data: "test-data"}}, + } + executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + ID: templateID, + Info: map[string]string{"severity": "low", "name": "test"}, + }) + err := request.Compile(executerOpts) + require.Nil(t, err, "could not compile network request") + + require.Equal(t, 2, len(request.addresses), "could not get correct number of input address") + t.Run("check-host", func(t *testing.T) { + require.Equal(t, "{{Hostname}}", request.addresses[0].key, "could not get correct host") + }) + t.Run("check-host-with-port", func(t *testing.T) { + require.Equal(t, "{{Hostname}}", request.addresses[1].key, "could not get correct host with port") + require.Equal(t, "8082", request.addresses[1].value, "could not get correct port for host") + }) +} diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 2638228e..699ba035 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -11,7 +11,7 @@ import ( func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { partString := matcher.Part switch partString { - case "body", "all", "": + case "body", "raw", "all", "": partString = "data" } @@ -38,14 +38,9 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) // Extract performs extracting operation for a extractor on model and returns true or false. func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - part, ok := data[extractor.Part] - if !ok { - return nil - } - partString := part.(string) - + partString := extractor.Part switch partString { - case "body", "all": + case "body", "raw", "all", "": partString = "data" } @@ -66,14 +61,12 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext // responseToDSLMap converts a DNS response to a map for use in DSL matching func (r *Request) responseToDSLMap(req, resp string, host, matched string) output.InternalEvent { - data := make(output.InternalEvent, 4) + data := make(output.InternalEvent, 6) // Some data regarding the request metadata data["host"] = host data["matched"] = matched - if r.options.Options.JSONRequests { - data["request"] = req - } + data["request"] = req data["data"] = resp data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go new file mode 100644 index 00000000..4afd66ef --- /dev/null +++ b/v2/pkg/protocols/network/operators_test.go @@ -0,0 +1,202 @@ +package network + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/stretchr/testify/require" +) + +func TestResponseToDSLMap(t *testing.T) { + options := testutils.DefaultOptions + + testutils.Init(options) + templateID := "testing-network" + request := &Request{ + ID: templateID, + Address: []string{"{{Hostname}}"}, + ReadSize: 1024, + Inputs: []*Input{&Input{Data: "test-data\r\n"}}, + } + executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + ID: templateID, + Info: map[string]string{"severity": "low", "name": "test"}, + }) + err := request.Compile(executerOpts) + require.Nil(t, err, "could not compile network request") + + req := "test-data\r\n" + resp := "resp-data\r\n" + event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one") + require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Equal(t, resp, event["data"], "could not get correct resp") +} + +func TestNetworkOperatorMatch(t *testing.T) { + options := testutils.DefaultOptions + + testutils.Init(options) + templateID := "testing-network" + request := &Request{ + ID: templateID, + Address: []string{"{{Hostname}}"}, + ReadSize: 1024, + Inputs: []*Input{&Input{Data: "test-data\r\n"}}, + } + executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + ID: templateID, + Info: map[string]string{"severity": "low", "name": "test"}, + }) + err := request.Compile(executerOpts) + require.Nil(t, err, "could not compile network request") + + req := "test-data\r\n" + resp := "resp-data\r\nSTAT \r\n" + event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one") + + t.Run("valid", func(t *testing.T) { + matcher := &matchers.Matcher{ + Part: "body", + Type: "word", + Words: []string{"STAT "}, + } + err = matcher.CompileMatchers() + require.Nil(t, err, "could not compile matcher") + + matched := request.Match(event, matcher) + require.True(t, matched, "could not match valid response") + }) + + t.Run("negative", func(t *testing.T) { + matcher := &matchers.Matcher{ + Part: "raw", + Type: "word", + Negative: true, + Words: []string{"random"}, + } + err := matcher.CompileMatchers() + require.Nil(t, err, "could not compile negative matcher") + + matched := request.Match(event, matcher) + require.True(t, matched, "could not match valid negative response matcher") + }) + + t.Run("invalid", func(t *testing.T) { + matcher := &matchers.Matcher{ + Part: "raw", + Type: "word", + Words: []string{"random"}, + } + err := matcher.CompileMatchers() + require.Nil(t, err, "could not compile matcher") + + matched := request.Match(event, matcher) + require.False(t, matched, "could match invalid response matcher") + }) +} + +func TestNetworkOperatorExtract(t *testing.T) { + options := testutils.DefaultOptions + + testutils.Init(options) + templateID := "testing-network" + request := &Request{ + ID: templateID, + Address: []string{"{{Hostname}}"}, + ReadSize: 1024, + Inputs: []*Input{&Input{Data: "test-data\r\n"}}, + } + executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + ID: templateID, + Info: map[string]string{"severity": "low", "name": "test"}, + }) + err := request.Compile(executerOpts) + require.Nil(t, err, "could not compile network request") + + req := "test-data\r\n" + resp := "resp-data\r\nSTAT \r\n1.1.1.1\r\n" + event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one") + + t.Run("extract", func(t *testing.T) { + extractor := &extractors.Extractor{ + Part: "raw", + Type: "regex", + Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, + } + err = extractor.CompileExtractors() + require.Nil(t, err, "could not compile extractor") + + data := request.Extract(event, extractor) + require.Greater(t, len(data), 0, "could not extractor valid response") + require.Equal(t, map[string]struct{}{"1.1.1.1": {}}, data, "could not extract correct data") + }) + + t.Run("kval", func(t *testing.T) { + extractor := &extractors.Extractor{ + Type: "kval", + KVal: []string{"request"}, + } + err = extractor.CompileExtractors() + require.Nil(t, err, "could not compile kval extractor") + + data := request.Extract(event, extractor) + require.Greater(t, len(data), 0, "could not extractor kval valid response") + require.Equal(t, map[string]struct{}{req: {}}, data, "could not extract correct kval data") + }) +} + +func TestNetworkMakeResult(t *testing.T) { + options := testutils.DefaultOptions + + testutils.Init(options) + templateID := "testing-network" + request := &Request{ + ID: templateID, + Address: []string{"{{Hostname}}"}, + ReadSize: 1024, + Inputs: []*Input{&Input{Data: "test-data\r\n"}}, + Operators: operators.Operators{ + Matchers: []*matchers.Matcher{ + &matchers.Matcher{ + Name: "test", + Part: "raw", + Type: "word", + Words: []string{"STAT "}, + }, + }, + Extractors: []*extractors.Extractor{ + &extractors.Extractor{ + Part: "raw", + Type: "regex", + Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, + }, + }, + }, + } + executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + ID: templateID, + Info: map[string]string{"severity": "low", "name": "test"}, + }) + err := request.Compile(executerOpts) + require.Nil(t, err, "could not compile network request") + + req := "test-data\r\n" + resp := "resp-data\rSTAT \r\n1.1.1.1\n" + event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one") + finalEvent := &output.InternalWrappedEvent{InternalEvent: event} + event["ip"] = "192.168.1.1" + if request.CompiledOperators != nil { + result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract) + if ok && result != nil { + finalEvent.OperatorsResult = result + finalEvent.Results = request.MakeResultEvent(finalEvent) + } + } + require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results") + require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results") + require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results") +} diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go new file mode 100644 index 00000000..7e8366da --- /dev/null +++ b/v2/pkg/protocols/network/request_test.go @@ -0,0 +1,101 @@ +package network + +import ( + "encoding/hex" + "testing" + + "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/stretchr/testify/require" +) + +func TestDNSExecuteWithResults(t *testing.T) { + options := testutils.DefaultOptions + + testutils.Init(options) + templateID := "testing-network" + request := &Request{ + ID: templateID, + Address: []string{"{{Hostname}}:80"}, + ReadSize: 2048, + Inputs: []*Input{&Input{Data: "GET / HTTP/1.1\r\n\r\n"}}, + Operators: operators.Operators{ + Matchers: []*matchers.Matcher{ + &matchers.Matcher{ + Name: "test", + Part: "raw", + Type: "word", + Words: []string{"400 - Bad Request"}, + }, + }, + Extractors: []*extractors.Extractor{ + &extractors.Extractor{ + Part: "raw", + Type: "regex", + Regex: []string{"

.*

"}, + }, + }, + }, + } + executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + ID: templateID, + Info: map[string]string{"severity": "low", "name": "test"}, + }) + err := request.Compile(executerOpts) + require.Nil(t, err, "could not compile network request") + + var finalEvent *output.InternalWrappedEvent + t.Run("domain-valid", func(t *testing.T) { + metadata := make(output.InternalEvent) + previous := make(output.InternalEvent) + err := request.ExecuteWithResults("example.com", metadata, previous, func(event *output.InternalWrappedEvent) { + finalEvent = event + }) + require.Nil(t, err, "could not execute network request") + }) + require.NotNil(t, finalEvent, "could not get event output from request") + require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results") + require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results") + require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results") + require.Equal(t, "

400 - Bad Request

", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results") + finalEvent = nil + + t.Run("invalid-port-override", func(t *testing.T) { + metadata := make(output.InternalEvent) + previous := make(output.InternalEvent) + err := request.ExecuteWithResults("example.com:11211", metadata, previous, func(event *output.InternalWrappedEvent) { + finalEvent = event + }) + require.Nil(t, err, "could not execute network request") + }) + require.NotNil(t, finalEvent, "could not get event output from request") + require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results") + require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results") + require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results") + require.Equal(t, "

400 - Bad Request

", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results") + finalEvent = nil + + request.Inputs[0].Type = "hex" + request.Inputs[0].Data = hex.EncodeToString([]byte("GET / HTTP/1.1\r\n\r\n")) + + t.Run("hex-to-string", func(t *testing.T) { + metadata := make(output.InternalEvent) + previous := make(output.InternalEvent) + err := request.ExecuteWithResults("example.com", metadata, previous, func(event *output.InternalWrappedEvent) { + finalEvent = event + }) + require.Nil(t, err, "could not execute network request") + }) + require.NotNil(t, finalEvent, "could not get event output from request") + require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results") + require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results") + require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results") + require.Equal(t, "

400 - Bad Request

", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results") + finalEvent = nil + + request.Inputs[0].Type = "" + request.Inputs[0].Data = "GET / HTTP/1.1\r\n\r\n" +}