From deaa86f5cc3beb500d7bfcd2aad8078c07bac4ec Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 16 Sep 2021 21:29:58 +0530 Subject: [PATCH 001/146] Added interactsh based template to integration test --- integration_tests/http/interactsh.yaml | 19 +++++++++++++++++ v2/cmd/integration-test/http.go | 29 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 integration_tests/http/interactsh.yaml diff --git a/integration_tests/http/interactsh.yaml b/integration_tests/http/interactsh.yaml new file mode 100644 index 00000000..28d9c560 --- /dev/null +++ b/integration_tests/http/interactsh.yaml @@ -0,0 +1,19 @@ +id: interactsh-integration-test + +info: + name: Interactsh Integration Test + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + headers: + url: 'http://{{interactsh-url}}' + + matchers: + - type: word + part: interactsh_protocol # Confirms the HTTP Interaction + words: + - "http" \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 41f177d2..37f1fadf 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -32,6 +32,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{}, "http/request-condition.yaml": &httpRequestCondition{}, "http/request-condition-new.yaml": &httpRequestCondition{}, + "http/interactsh.yaml": &httpInteractshRequest{}, } func httpDebugRequestDump(r *http.Request) { @@ -42,6 +43,34 @@ func httpDebugRequestDump(r *http.Request) { } } +type httpInteractshRequest struct{} + +// Executes executes a test case and returns an error if occurred +func (h *httpInteractshRequest) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + httpDebugRequestDump(r) + + value := r.Header.Get("url") + if value != "" { + if resp, _ := http.DefaultClient.Get(value); resp != nil { + resp.Body.Close() + } + } + })) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + type httpGetHeaders struct{} // Executes executes a test case and returns an error if occurred From 5c31b75ac79af453615aaf9cc3279d7496592aca Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 22 Sep 2021 22:41:07 +0530 Subject: [PATCH 002/146] Added ssl protocol to nuclei --- v2/pkg/operators/common/dsl/dsl.go | 116 +++++++++++++ v2/pkg/protocols/others/others.go | 1 + v2/pkg/protocols/others/ssl/ssl.go | 208 ++++++++++++++++++++++++ v2/pkg/protocols/others/ssl/ssl_test.go | 28 ++++ v2/pkg/templates/compile.go | 64 +++++--- v2/pkg/templates/templates.go | 4 + 6 files changed, 397 insertions(+), 24 deletions(-) create mode 100644 v2/pkg/protocols/others/others.go create mode 100644 v2/pkg/protocols/others/ssl/ssl.go create mode 100644 v2/pkg/protocols/others/ssl/ssl_test.go diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 056f4ace..0b99505e 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -31,21 +31,38 @@ const ( withMaxRandArgsSize = withCutSetArgsSize ) +var ErrDSLArguments = errors.New("invalid arguments provided to dsl") + var functions = map[string]govaluate.ExpressionFunction{ "len": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } length := len(types.ToString(args[0])) return float64(length), nil }, "toupper": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return strings.ToUpper(types.ToString(args[0])), nil }, "tolower": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return strings.ToLower(types.ToString(args[0])), nil }, "replace": func(args ...interface{}) (interface{}, error) { + if len(args) != 3 { + return nil, ErrDSLArguments + } return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil }, "replace_regex": func(args ...interface{}) (interface{}, error) { + if len(args) != 3 { + return nil, ErrDSLArguments + } compiled, err := regexp.Compile(types.ToString(args[1])) if err != nil { return nil, err @@ -53,66 +70,120 @@ var functions = map[string]govaluate.ExpressionFunction{ return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil }, "trim": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } return strings.Trim(types.ToString(args[0]), types.ToString(args[1])), nil }, "trimleft": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil }, "trimright": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil }, "trimspace": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return strings.TrimSpace(types.ToString(args[0])), nil }, "trimprefix": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil }, "trimsuffix": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil }, "reverse": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return reverseString(types.ToString(args[0])), nil }, // encoding "base64": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))) return sEnc, nil }, // python encodes to base64 with lines of 76 bytes terminated by new line "\n" "base64_py": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))) return deserialization.InsertInto(sEnc, 76, '\n'), nil }, "base64_decode": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return base64.StdEncoding.DecodeString(types.ToString(args[0])) }, "url_encode": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return url.QueryEscape(types.ToString(args[0])), nil }, "url_decode": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return url.QueryUnescape(types.ToString(args[0])) }, "hex_encode": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return hex.EncodeToString([]byte(types.ToString(args[0]))), nil }, "hex_decode": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } hx, _ := hex.DecodeString(types.ToString(args[0])) return string(hx), nil }, "html_escape": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return html.EscapeString(types.ToString(args[0])), nil }, "html_unescape": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return html.UnescapeString(types.ToString(args[0])), nil }, // hashing "md5": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } hash := md5.Sum([]byte(types.ToString(args[0]))) return hex.EncodeToString(hash[:]), nil }, "sha256": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } h := sha256.New() if _, err := h.Write([]byte(types.ToString(args[0]))); err != nil { return nil, err @@ -120,6 +191,9 @@ var functions = map[string]govaluate.ExpressionFunction{ return hex.EncodeToString(h.Sum(nil)), nil }, "sha1": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } h := sha1.New() if _, err := h.Write([]byte(types.ToString(args[0]))); err != nil { return nil, err @@ -127,13 +201,22 @@ var functions = map[string]govaluate.ExpressionFunction{ return hex.EncodeToString(h.Sum(nil)), nil }, "mmh3": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil }, // search "contains": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil }, "regex": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } compiled, err := regexp.Compile(types.ToString(args[0])) if err != nil { return nil, err @@ -142,6 +225,9 @@ var functions = map[string]govaluate.ExpressionFunction{ }, // random generators "rand_char": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } chars := letters + numbers bad := "" if len(args) >= 1 { @@ -154,6 +240,9 @@ var functions = map[string]govaluate.ExpressionFunction{ return chars[rand.Intn(len(chars))], nil }, "rand_base": func(args ...interface{}) (interface{}, error) { + if len(args) != 3 { + return nil, ErrDSLArguments + } l := 0 bad := "" base := letters + numbers @@ -171,6 +260,9 @@ var functions = map[string]govaluate.ExpressionFunction{ return randSeq(base, l), nil }, "rand_text_alphanumeric": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } l := 0 bad := "" chars := letters + numbers @@ -185,6 +277,9 @@ var functions = map[string]govaluate.ExpressionFunction{ return randSeq(chars, l), nil }, "rand_text_alpha": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } l := 0 bad := "" chars := letters @@ -199,6 +294,9 @@ var functions = map[string]govaluate.ExpressionFunction{ return randSeq(chars, l), nil }, "rand_text_numeric": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } l := 0 bad := "" chars := numbers @@ -213,6 +311,9 @@ var functions = map[string]govaluate.ExpressionFunction{ return randSeq(chars, l), nil }, "rand_int": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } min := 0 max := math.MaxInt32 @@ -226,12 +327,18 @@ var functions = map[string]govaluate.ExpressionFunction{ }, // Time Functions "waitfor": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } seconds := args[0].(float64) time.Sleep(time.Duration(seconds) * time.Second) return true, nil }, // deserialization Functions "generate_java_gadget": func(args ...interface{}) (interface{}, error) { + if len(args) != 3 { + return nil, ErrDSLArguments + } gadget := args[0].(string) cmd := args[1].(string) @@ -247,6 +354,15 @@ var functions = map[string]govaluate.ExpressionFunction{ gologger.Info().Msgf("print_debug value: %s", fmt.Sprint(args)) return true, nil }, + // is_before_now compares a timestamp and returns true if the first + // passed argument is a time.Time that has already passed. + "is_time_before_now": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } + isBefore := time.Since(args[0].(time.Time)) > 0 + return isBefore, nil + }, } // HelperFunctions returns the dsl helper functions diff --git a/v2/pkg/protocols/others/others.go b/v2/pkg/protocols/others/others.go new file mode 100644 index 00000000..24b5bd24 --- /dev/null +++ b/v2/pkg/protocols/others/others.go @@ -0,0 +1 @@ +package others diff --git a/v2/pkg/protocols/others/ssl/ssl.go b/v2/pkg/protocols/others/ssl/ssl.go new file mode 100644 index 00000000..09fb2301 --- /dev/null +++ b/v2/pkg/protocols/others/ssl/ssl.go @@ -0,0 +1,208 @@ +package ssl + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/url" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Request is a request for the SSL protocol +type Request struct { + // Operators for the current request go here. + operators.Operators `yaml:",inline,omitempty"` + CompiledOperators *operators.Operators `yaml:"-"` + + // cache any variables that may be needed for operation. + dialer *fastdialer.Dialer + options *protocols.ExecuterOptions +} + +// Compile compiles the request generators preparing any requests possible. +func (r *Request) Compile(options *protocols.ExecuterOptions) error { + r.options = options + + client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) + if err != nil { + return errors.Wrap(err, "could not get network client") + } + r.dialer = client + + if len(r.Matchers) > 0 || len(r.Extractors) > 0 { + compiled := &r.Operators + if err := compiled.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + r.CompiledOperators = compiled + } + return nil +} + +// Requests returns the total number of requests the rule will perform +func (r *Request) Requests() int { + return 1 +} + +// GetID returns the ID for the request if any. +func (r *Request) GetID() string { + return "" +} + +// Match performs matching operation for a matcher on model and returns true or false. +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { + partItem, ok := data[matcher.Part] + if !ok { + return false + } + item := types.ToString(partItem) + + switch matcher.GetType() { + case matchers.SizeMatcher: + return matcher.Result(matcher.MatchSize(len(item))) + case matchers.WordsMatcher: + return matcher.Result(matcher.MatchWords(item)) + case matchers.RegexMatcher: + return matcher.Result(matcher.MatchRegex(item)) + case matchers.BinaryMatcher: + return matcher.Result(matcher.MatchBinary(item)) + case matchers.DSLMatcher: + return matcher.Result(matcher.MatchDSL(data)) + } + return false +} + +// Extract performs extracting operation for an extractor on model and returns true or false. +func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { + item, ok := data[extractor.Part] + if !ok { + return nil + } + itemStr := types.ToString(item) + + switch extractor.GetType() { + case extractors.RegexExtractor: + return extractor.ExtractRegex(itemStr) + case extractors.KValExtractor: + return extractor.ExtractKval(data) + } + return nil +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + address, err := getAddress(input) + if err != nil { + return nil + } + fmt.Printf("address: %v\n", address) + hostname, _, _ := net.SplitHostPort(input) + + config := &tls.Config{InsecureSkipVerify: true, ServerName: hostname} + conn, err := r.dialer.DialTLSWithConfig(context.Background(), "tcp", address, config) + if err != nil { + r.options.Output.Request(r.options.TemplateID, input, "ssl", err) + r.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not connect to server") + } + defer conn.Close() + _ = conn.SetReadDeadline(time.Now().Add(time.Duration(r.options.Options.Timeout) * time.Second)) + + connTLS, ok := conn.(*tls.Conn) + if !ok { + return nil + } + if len(connTLS.ConnectionState().PeerCertificates) == 0 { + return nil + } + data := make(map[string]interface{}) + cert := connTLS.ConnectionState().PeerCertificates[0] + data["host"] = input + data["not_after"] = cert.NotAfter + data["ip"] = r.dialer.GetDialedIP(hostname) + + event := &output.InternalWrappedEvent{InternalEvent: data} + if r.CompiledOperators != nil { + var ok bool + event.OperatorsResult, ok = r.CompiledOperators.Execute(data, r.Match, r.Extract) + if ok && event.OperatorsResult != nil { + event.Results = r.makeResultEvent(event) + } + callback(event) + } + return nil +} + +// getAddress returns the address of the host to make request to +func getAddress(toTest string) (string, error) { + if strings.Contains(toTest, "://") { + parsed, err := url.Parse(toTest) + if err != nil { + return "", err + } + _, port, _ := net.SplitHostPort(parsed.Host) + + if parsed.Scheme == "https" && port == "" { + toTest = net.JoinHostPort(parsed.Host, "443") + } else { + toTest = parsed.Host + } + } + return toTest, nil +} + +// makeResultEvent creates a result event from internal wrapped event +func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + if len(wrapped.OperatorsResult.DynamicValues) > 0 { + return nil + } + results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) + + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data := r.makeResultEventItem(wrapped) + data.MatcherName = k + results = append(results, data) + } + } else if len(wrapped.OperatorsResult.Extracts) > 0 { + for k, v := range wrapped.OperatorsResult.Extracts { + data := r.makeResultEventItem(wrapped) + data.ExtractedResults = v + data.ExtractorName = k + results = append(results, data) + } + } else { + data := r.makeResultEventItem(wrapped) + results = append(results, data) + } + return results +} + +func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ + TemplateID: types.ToString(r.options.TemplateID), + TemplatePath: types.ToString(r.options.TemplatePath), + Info: r.options.TemplateInfo, + Type: "ssl", + Host: types.ToString(wrapped.InternalEvent["host"]), + Matched: types.ToString(wrapped.InternalEvent["host"]), + Metadata: wrapped.OperatorsResult.PayloadValues, + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + Timestamp: time.Now(), + IP: types.ToString(wrapped.InternalEvent["ip"]), + } + return data +} diff --git a/v2/pkg/protocols/others/ssl/ssl_test.go b/v2/pkg/protocols/others/ssl/ssl_test.go new file mode 100644 index 00000000..a043518a --- /dev/null +++ b/v2/pkg/protocols/others/ssl/ssl_test.go @@ -0,0 +1,28 @@ +package ssl + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/stretchr/testify/require" +) + +func TestSSLProtocol(t *testing.T) { + options := testutils.DefaultOptions + + testutils.Init(options) + templateID := "testing-ssl" + request := &Request{} + executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + ID: templateID, + Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, + }) + err := request.Compile(executerOpts) + require.Nil(t, err, "could not compile ssl request") + + err = request.ExecuteWithResults("google.com:443", nil, nil, func(event *output.InternalWrappedEvent) {}) + require.Nil(t, err, "could not run ssl request") +} diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index d7f545bc..72d88062 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -70,7 +70,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute options.TemplatePath = filePath // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows) == 0 { + if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows)+len(template.RequestsSSL) == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) } @@ -85,12 +85,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute // Compile the requests found requests := []protocols.Request{} - if len(template.RequestsDNS) > 0 && !options.Options.OfflineHTTP { - for _, req := range template.RequestsDNS { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } + if len(template.RequestsHTTP) > 0 { if options.Options.OfflineHTTP { operatorsList := []*operators.Operators{} @@ -115,24 +110,10 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute template.Executer = executer.NewExecuter(requests, &options) } } - if len(template.RequestsFile) > 0 && !options.Options.OfflineHTTP { - for _, req := range template.RequestsFile { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - if len(template.RequestsNetwork) > 0 && !options.Options.OfflineHTTP { - for _, req := range template.RequestsNetwork { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - if len(template.RequestsHeadless) > 0 && !options.Options.OfflineHTTP && options.Options.Headless { - for _, req := range template.RequestsHeadless { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) + if !options.Options.OfflineHTTP { + makeRequestsForTemplate(template, options) } + if template.Executer != nil { if err := template.Executer.Compile(); err != nil { return nil, errors.Wrap(err, "could not compile request") @@ -147,3 +128,38 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute parsedTemplatesCache.Store(filePath, template, err) return template, nil } + +func makeRequestsForTemplate(template *Template, options protocols.ExecuterOptions) { + requests := []protocols.Request{} + + if len(template.RequestsDNS) > 0 { + for _, req := range template.RequestsDNS { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, &options) + } + if len(template.RequestsFile) > 0 { + for _, req := range template.RequestsFile { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, &options) + } + if len(template.RequestsNetwork) > 0 { + for _, req := range template.RequestsNetwork { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, &options) + } + if len(template.RequestsHeadless) > 0 && options.Options.Headless { + for _, req := range template.RequestsHeadless { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, &options) + } + if len(template.RequestsSSL) > 0 { + for _, req := range template.RequestsSSL { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, &options) + } +} diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index b6f2491a..6a19d2e5 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/ssl" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) @@ -56,6 +57,9 @@ type Template struct { // description: | // Headless contains the headless request to make in the template. RequestsHeadless []*headless.Request `yaml:"headless,omitempty" json:"headless,omitempty" jsonschema:"title=headless requests to make,description=Headless requests to make for the template"` + // description: | + // SSL contains the SSL request to make in the template. + RequestsSSL []*ssl.Request `yaml:"ssl,omitempty" json:"ssl,omitempty" jsonschema:"title=ssl requests to make,description=SSL requests to make for the template"` // description: | // Workflows is a yaml based workflow declaration code. From f6e9acf06f8295a9ecfd215574088cf8da481fb5 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 23 Sep 2021 18:30:44 +0530 Subject: [PATCH 003/146] More work on other protocol additions --- v2/go.mod | 2 +- v2/go.sum | 2 + v2/pkg/protocols/others/ssl/ssl.go | 77 +----------------------- v2/pkg/protocols/others/utils/utils.go | 83 ++++++++++++++++++++++++++ v2/pkg/protocols/others/wss/wss.go | 1 + v2/pkg/protocols/protocols.go | 6 -- v2/pkg/templates/compile.go | 15 ++++- 7 files changed, 104 insertions(+), 82 deletions(-) create mode 100644 v2/pkg/protocols/others/utils/utils.go create mode 100644 v2/pkg/protocols/others/wss/wss.go diff --git a/v2/go.mod b/v2/go.mod index 3f550246..3a3e5eb3 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -30,7 +30,7 @@ require ( github.com/owenrumney/go-sarif v1.0.11 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 - github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e + github.com/projectdiscovery/fastdialer v0.0.13-0.20210923125921-675fa1873feb github.com/projectdiscovery/goflags v0.0.7 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa diff --git a/v2/go.sum b/v2/go.sum index d908b8fe..82134aed 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -341,6 +341,8 @@ github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ= +github.com/projectdiscovery/fastdialer v0.0.13-0.20210923125921-675fa1873feb h1:h+HvVw51KUvcO4Tww1QCd95D6MWV/6wpXuSbmFpPQSI= +github.com/projectdiscovery/fastdialer v0.0.13-0.20210923125921-675fa1873feb/go.mod h1:Mex24omi3RxrmhA8Ote7rw+6LWMiaBvbJq8CNp0ksII= github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= diff --git a/v2/pkg/protocols/others/ssl/ssl.go b/v2/pkg/protocols/others/ssl/ssl.go index 09fb2301..6a33f27e 100644 --- a/v2/pkg/protocols/others/ssl/ssl.go +++ b/v2/pkg/protocols/others/ssl/ssl.go @@ -3,7 +3,6 @@ package ssl import ( "context" "crypto/tls" - "fmt" "net" "net/url" "strings" @@ -12,11 +11,10 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/nuclei/v2/pkg/operators" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/utils" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -61,53 +59,12 @@ func (r *Request) GetID() string { return "" } -// Match performs matching operation for a matcher on model and returns true or false. -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { - partItem, ok := data[matcher.Part] - if !ok { - return false - } - item := types.ToString(partItem) - - switch matcher.GetType() { - case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(item))) - case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(item)) - case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(item)) - case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(item)) - case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) - } - return false -} - -// Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - item, ok := data[extractor.Part] - if !ok { - return nil - } - itemStr := types.ToString(item) - - switch extractor.GetType() { - case extractors.RegexExtractor: - return extractor.ExtractRegex(itemStr) - case extractors.KValExtractor: - return extractor.ExtractKval(data) - } - return nil -} - // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (r *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { address, err := getAddress(input) if err != nil { return nil } - fmt.Printf("address: %v\n", address) hostname, _, _ := net.SplitHostPort(input) config := &tls.Config{InsecureSkipVerify: true, ServerName: hostname} @@ -136,9 +93,9 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu event := &output.InternalWrappedEvent{InternalEvent: data} if r.CompiledOperators != nil { var ok bool - event.OperatorsResult, ok = r.CompiledOperators.Execute(data, r.Match, r.Extract) + event.OperatorsResult, ok = r.CompiledOperators.Execute(data, utils.MatchFunc, utils.ExtractFunc) if ok && event.OperatorsResult != nil { - event.Results = r.makeResultEvent(event) + event.Results = utils.MakeResultEvent(event, r.makeResultEventItem) } callback(event) } @@ -163,34 +120,6 @@ func getAddress(toTest string) (string, error) { return toTest, nil } -// makeResultEvent creates a result event from internal wrapped event -func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := r.makeResultEventItem(wrapped) - results = append(results, data) - } - return results -} - func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(r.options.TemplateID), diff --git a/v2/pkg/protocols/others/utils/utils.go b/v2/pkg/protocols/others/utils/utils.go new file mode 100644 index 00000000..804d0a94 --- /dev/null +++ b/v2/pkg/protocols/others/utils/utils.go @@ -0,0 +1,83 @@ +package utils + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// MakeResultEventItemFunc returns a result event for an internal wrapped event item +type MakeResultEventItemFunc func(wrapped *output.InternalWrappedEvent) *output.ResultEvent + +// MakeResultEvent creates a result event from internal wrapped event +func MakeResultEvent(wrapped *output.InternalWrappedEvent, makeEventItemFunc MakeResultEventItemFunc) []*output.ResultEvent { + if len(wrapped.OperatorsResult.DynamicValues) > 0 { + return nil + } + results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) + + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data := makeEventItemFunc(wrapped) + data.MatcherName = k + results = append(results, data) + } + } else if len(wrapped.OperatorsResult.Extracts) > 0 { + for k, v := range wrapped.OperatorsResult.Extracts { + data := makeEventItemFunc(wrapped) + data.ExtractedResults = v + data.ExtractorName = k + results = append(results, data) + } + } else { + data := makeEventItemFunc(wrapped) + results = append(results, data) + } + return results +} + +// ExtractFunc performs extracting operation for an extractor on model and returns true or false. +func ExtractFunc(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { + item, ok := data[extractor.Part] + if !ok { + return nil + } + itemStr := types.ToString(item) + + switch extractor.GetType() { + case extractors.RegexExtractor: + return extractor.ExtractRegex(itemStr) + case extractors.KValExtractor: + return extractor.ExtractKval(data) + case extractors.JSONExtractor: + return extractor.ExtractJSON(itemStr) + case extractors.XPathExtractor: + return extractor.ExtractHTML(itemStr) + } + return nil +} + +// MatchFunc performs matching operation for a matcher on model and returns true or false. +func MatchFunc(data map[string]interface{}, matcher *matchers.Matcher) bool { + partItem, ok := data[matcher.Part] + if !ok { + return false + } + item := types.ToString(partItem) + + switch matcher.GetType() { + case matchers.SizeMatcher: + return matcher.Result(matcher.MatchSize(len(item))) + case matchers.WordsMatcher: + return matcher.Result(matcher.MatchWords(item)) + case matchers.RegexMatcher: + return matcher.Result(matcher.MatchRegex(item)) + case matchers.BinaryMatcher: + return matcher.Result(matcher.MatchBinary(item)) + case matchers.DSLMatcher: + return matcher.Result(matcher.MatchDSL(data)) + } + return false +} diff --git a/v2/pkg/protocols/others/wss/wss.go b/v2/pkg/protocols/others/wss/wss.go new file mode 100644 index 00000000..c4004307 --- /dev/null +++ b/v2/pkg/protocols/others/wss/wss.go @@ -0,0 +1 @@ +package wss diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 9ca38333..137cdfb6 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -6,8 +6,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/operators" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" @@ -74,10 +72,6 @@ type Request interface { // condition matching. So, two requests can be sent and their match can // be evaluated from the third request by using the IDs for both requests. GetID() string - // Match performs matching operation for a matcher on model and returns true or false. - Match(data map[string]interface{}, matcher *matchers.Matcher) bool - // Extract performs extracting operation for an extractor on model and returns true or false. - Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error } diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 72d88062..bd711492 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -70,7 +70,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute options.TemplatePath = filePath // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows)+len(template.RequestsSSL) == 0 { + if template.Requests() == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) } @@ -129,6 +129,19 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute return template, nil } +// Requests returns the total number of requests for the template. +func (t *Template) Requests() int { + sum := len(t.RequestsDNS) + + len(t.RequestsHTTP) + + len(t.RequestsFile) + + len(t.RequestsNetwork) + + len(t.RequestsHeadless) + + len(t.Workflows) + + len(t.RequestsSSL) + return sum +} + +// makeRequestsForTemplate compiles all the requests for the template. func makeRequestsForTemplate(template *Template, options protocols.ExecuterOptions) { requests := []protocols.Request{} From 0b11b80d8afb997cb879b702b88300865caa8ccc Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 24 Sep 2021 19:35:00 +0530 Subject: [PATCH 004/146] Fixed some bugs with ssl protocols + misc enhancements --- v2/pkg/operators/common/dsl/dsl.go | 8 ++------ v2/pkg/operators/common/dsl/dsl_test.go | 11 +++++++++++ v2/pkg/protocols/others/ssl/ssl.go | 9 +++++++-- v2/pkg/protocols/others/ssl/ssl_test.go | 5 +++++ v2/pkg/protocols/others/utils/utils.go | 2 +- 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 0b99505e..6bff036a 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -356,12 +356,8 @@ var functions = map[string]govaluate.ExpressionFunction{ }, // is_before_now compares a timestamp and returns true if the first // passed argument is a time.Time that has already passed. - "is_time_before_now": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - isBefore := time.Since(args[0].(time.Time)) > 0 - return isBefore, nil + "time_now": func(args ...interface{}) (interface{}, error) { + return float64(time.Now().Unix()), nil }, } diff --git a/v2/pkg/operators/common/dsl/dsl_test.go b/v2/pkg/operators/common/dsl/dsl_test.go index bf2c5bfe..42bf5810 100644 --- a/v2/pkg/operators/common/dsl/dsl_test.go +++ b/v2/pkg/operators/common/dsl/dsl_test.go @@ -2,7 +2,9 @@ package dsl import ( "testing" + "time" + "github.com/Knetic/govaluate" "github.com/stretchr/testify/require" ) @@ -17,3 +19,12 @@ func TestDSLURLEncodeDecode(t *testing.T) { require.Nil(t, err, "could not url encode") require.Equal(t, "&test\"", decoded, "could not get url decoded data") } + +func TestDSLTimeComparison(t *testing.T) { + compiled, err := govaluate.NewEvaluableExpressionWithFunctions("time_now() > not_after", HelperFunctions()) + require.Nil(t, err, "could not compare time") + + result, err := compiled.Evaluate(map[string]interface{}{"not_after": float64(time.Now().Unix() - 1000)}) + require.Nil(t, err, "could not evaluate compare time") + require.Equal(t, true, result, "could not get url encoded data") +} diff --git a/v2/pkg/protocols/others/ssl/ssl.go b/v2/pkg/protocols/others/ssl/ssl.go index 6a33f27e..ded0274d 100644 --- a/v2/pkg/protocols/others/ssl/ssl.go +++ b/v2/pkg/protocols/others/ssl/ssl.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" @@ -65,7 +66,7 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu if err != nil { return nil } - hostname, _, _ := net.SplitHostPort(input) + hostname, _, _ := net.SplitHostPort(address) config := &tls.Config{InsecureSkipVerify: true, ServerName: hostname} conn, err := r.dialer.DialTLSWithConfig(context.Background(), "tcp", address, config) @@ -81,13 +82,16 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu if !ok { return nil } + r.options.Output.Request(r.options.TemplateID, address, "ssl", err) + gologger.Verbose().Msgf("Sent SSL request to %s", address) + if len(connTLS.ConnectionState().PeerCertificates) == 0 { return nil } data := make(map[string]interface{}) cert := connTLS.ConnectionState().PeerCertificates[0] data["host"] = input - data["not_after"] = cert.NotAfter + data["not_after"] = float64(cert.NotAfter.Unix()) data["ip"] = r.dialer.GetDialedIP(hostname) event := &output.InternalWrappedEvent{InternalEvent: data} @@ -116,6 +120,7 @@ func getAddress(toTest string) (string, error) { } else { toTest = parsed.Host } + return toTest, nil } return toTest, nil } diff --git a/v2/pkg/protocols/others/ssl/ssl_test.go b/v2/pkg/protocols/others/ssl/ssl_test.go index a043518a..fef44fbb 100644 --- a/v2/pkg/protocols/others/ssl/ssl_test.go +++ b/v2/pkg/protocols/others/ssl/ssl_test.go @@ -26,3 +26,8 @@ func TestSSLProtocol(t *testing.T) { err = request.ExecuteWithResults("google.com:443", nil, nil, func(event *output.InternalWrappedEvent) {}) require.Nil(t, err, "could not run ssl request") } + +func TestGetAddress(t *testing.T) { + address, _ := getAddress("https://google.com") + require.Equal(t, "google.com:443", address, "could not get correct address") +} diff --git a/v2/pkg/protocols/others/utils/utils.go b/v2/pkg/protocols/others/utils/utils.go index 804d0a94..89771ca9 100644 --- a/v2/pkg/protocols/others/utils/utils.go +++ b/v2/pkg/protocols/others/utils/utils.go @@ -62,7 +62,7 @@ func ExtractFunc(data map[string]interface{}, extractor *extractors.Extractor) m // MatchFunc performs matching operation for a matcher on model and returns true or false. func MatchFunc(data map[string]interface{}, matcher *matchers.Matcher) bool { partItem, ok := data[matcher.Part] - if !ok { + if !ok && len(matcher.DSL) == 0 { return false } item := types.ToString(partItem) From 396f17484ed3eac69e01285c4526339f1465676d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 27 Sep 2021 18:02:49 +0530 Subject: [PATCH 005/146] Added websocket protocol support to nuclei --- v2/go.mod | 4 + v2/go.sum | 9 + .../protocols/others/websocket/websocket.go | 300 ++++++++++++++++++ v2/pkg/protocols/others/wss/wss.go | 1 - v2/pkg/templates/compile.go | 9 +- v2/pkg/templates/templates.go | 4 + 6 files changed, 325 insertions(+), 2 deletions(-) create mode 100644 v2/pkg/protocols/others/websocket/websocket.go delete mode 100644 v2/pkg/protocols/others/wss/wss.go diff --git a/v2/go.mod b/v2/go.mod index 3a3e5eb3..db6b30da 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -79,12 +79,16 @@ require ( github.com/eggsampler/acme/v3 v3.2.1 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/go-ole/go-ole v1.2.5 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.1.0 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/uuid v1.2.0 // indirect + github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 82134aed..7e8ed14c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -143,6 +143,12 @@ github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8w github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI= github.com/go-rod/rod v0.101.7 h1:kbI5CNvcRhf7feybBln4xDutsM0mbsF0ENNZfKcF6WA= github.com/go-rod/rod v0.101.7/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= +github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -214,6 +220,8 @@ github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= @@ -637,6 +645,7 @@ golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/v2/pkg/protocols/others/websocket/websocket.go b/v2/pkg/protocols/others/websocket/websocket.go new file mode 100644 index 00000000..db62c663 --- /dev/null +++ b/v2/pkg/protocols/others/websocket/websocket.go @@ -0,0 +1,300 @@ +package websocket + +import ( + "context" + "crypto/tls" + "io" + "net/url" + "strings" + "time" + + "github.com/gobwas/ws" + "github.com/gobwas/ws/wsutil" + "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/utils" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Request is a request for the Websocket protocol +type Request struct { + // Operators for the current request go here. + operators.Operators `yaml:",inline,omitempty"` + CompiledOperators *operators.Operators `yaml:"-"` + + // description: | + // Inputs contains inputs for the websocket protocol + Inputs []*Input `yaml:"inputs,omitempty" jsonschema:"title=inputs for the websocket request,description=Inputs contains any input/output for the current request"` + + // description: | + // Attack is the type of payload combinations to perform. + // + // Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates + // permutations and combinations for all payloads. + // values: + // - "sniper" + // - "pitchfork" + // - "clusterbomb" + AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` + // description: | + // Payloads contains any payloads for the current request. + // + // Payloads support both key-values combinations where a list + // of payloads is provided, or optionally a single file can also + // be provided as payload which will be read on run-time. + Payloads map[string]interface{} `yaml:"payloads,omitempty" jsonschema:"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request"` + + generator *generators.Generator + attackType generators.Type + + // cache any variables that may be needed for operation. + dialer *fastdialer.Dialer + options *protocols.ExecuterOptions +} + +// Input is an input for the websocket protocol +type Input struct { + // description: | + // Data is the data to send as the input. + // + // It supports DSL Helper Functions as well as normal expressions. + // examples: + // - value: "\"TEST\"" + // - value: "\"hex_decode('50494e47')\"" + Data string `yaml:"data,omitempty" jsonschema:"title=data to send as input,description=Data is the data to send as the input"` + // description: | + // Name is the optional name of the data read to provide matching on. + // examples: + // - value: "\"prefix\"" + Name string `yaml:"name,omitempty" jsonschema:"title=optional name for data read,description=Optional name of the data read to provide matching on"` +} + +// Compile compiles the request generators preparing any requests possible. +func (r *Request) Compile(options *protocols.ExecuterOptions) error { + r.options = options + + client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) + if err != nil { + return errors.Wrap(err, "could not get network client") + } + r.dialer = client + + if len(r.Payloads) > 0 { + attackType := r.AttackType + if attackType == "" { + attackType = "sniper" + } + r.attackType = generators.StringToType[attackType] + + // Resolve payload paths if they are files. + for name, payload := range r.Payloads { + payloadStr, ok := payload.(string) + if ok { + final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) + if resolveErr != nil { + return errors.Wrap(resolveErr, "could not read payload file") + } + r.Payloads[name] = final + } + } + r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath) + if err != nil { + return errors.Wrap(err, "could not parse payloads") + } + } + + if len(r.Matchers) > 0 || len(r.Extractors) > 0 { + compiled := &r.Operators + if err := compiled.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + r.CompiledOperators = compiled + } + return nil +} + +// Requests returns the total number of requests the rule will perform +func (r *Request) Requests() int { + if r.generator != nil { + return r.generator.NewIterator().Total() + } + return 1 +} + +// GetID returns the ID for the request if any. +func (r *Request) GetID() string { + return "" +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + hostname, err := getAddress(input) + if err != nil { + return nil + } + + if r.generator != nil { + iterator := r.generator.NewIterator() + + for { + value, ok := iterator.Value() + if !ok { + break + } + if err := r.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { + return err + } + } + } else { + value := make(map[string]interface{}) + if err := r.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { + return err + } + } + return nil +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + websocketDialer := ws.Dialer{ + Timeout: time.Duration(r.options.Options.Timeout) * time.Second, + NetDial: r.dialer.Dial, + TLSConfig: &tls.Config{InsecureSkipVerify: true, ServerName: hostname}, + } + + conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), input) + if err != nil { + r.options.Output.Request(r.options.TemplateID, input, "ssl", err) + r.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not connect to server") + } + defer conn.Close() + + responseBuilder := &strings.Builder{} + if readBuffer != nil { + io.Copy(responseBuilder, readBuffer) // Copy initial response + } + + reqBuilder := &strings.Builder{} + + inputEvents := make(map[string]interface{}) + for _, req := range r.Inputs { + reqBuilder.Grow(len(req.Data)) + reqBuilder.WriteString(req.Data) + + finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), dynamicValues) + if dataErr != nil { + r.options.Output.Request(r.options.TemplateID, input, "websocket", dataErr) + r.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not evaluate template expressions") + } + err = wsutil.WriteClientMessage(conn, ws.OpText, finalData) + if err != nil { + r.options.Output.Request(r.options.TemplateID, input, "websocket", err) + r.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not write request to server") + } + + msg, _, err := wsutil.ReadServerData(conn) + if err != nil { + r.options.Output.Request(r.options.TemplateID, input, "websocket", err) + r.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not write request to server") + } + + responseBuilder.Write(msg) + if req.Name != "" { + bufferStr := string(msg) + if req.Name != "" { + inputEvents[req.Name] = bufferStr + } + + // Run any internal extractors for the request here and add found values to map. + if r.CompiledOperators != nil { + values := r.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{req.Name: bufferStr}, utils.ExtractFunc) + for k, v := range values { + dynamicValues[k] = v + } + } + } + } + r.options.Progress.IncrementRequests() + + if r.options.Options.Debug || r.options.Options.DebugRequests { + requestOutput := reqBuilder.String() + gologger.Info().Str("address", input).Msgf("[%s] Dumped Websocket request for %s", r.options.TemplateID, input) + gologger.Print().Msgf("%s", requestOutput) + } + + r.options.Output.Request(r.options.TemplateID, input, "websocket", err) + gologger.Verbose().Msgf("Sent Websocket request to %s", input) + + if r.options.Options.Debug || r.options.Options.DebugResponse { + responseOutput := responseBuilder.String() + gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", r.options.TemplateID, input) + gologger.Print().Msgf("%s", responseOutput) + } + + data := make(map[string]interface{}) + for k, v := range previous { + data[k] = v + } + for k, v := range dynamicValues { + data[k] = v + } + for k, v := range inputEvents { + data[k] = v + } + data["request"] = reqBuilder.String() + data["response"] = responseBuilder.String() + data["host"] = input + data["ip"] = r.dialer.GetDialedIP(hostname) + + event := &output.InternalWrappedEvent{InternalEvent: data} + if r.CompiledOperators != nil { + var ok bool + event.OperatorsResult, ok = r.CompiledOperators.Execute(data, utils.MatchFunc, utils.ExtractFunc) + if ok && event.OperatorsResult != nil { + event.Results = utils.MakeResultEvent(event, r.makeResultEventItem) + } + callback(event) + } + return nil +} + +// getAddress returns the address of the host to make request to +func getAddress(toTest string) (string, error) { + if !strings.HasPrefix(toTest, "ws://") && !strings.HasPrefix(toTest, "wss://") { + return "", errors.New("invalid websocket provided") + } + parsed, _ := url.Parse(toTest) + if parsed != nil && parsed.Host != "" { + return parsed.Host, nil + } + return "", nil +} + +func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ + TemplateID: types.ToString(r.options.TemplateID), + TemplatePath: types.ToString(r.options.TemplatePath), + Info: r.options.TemplateInfo, + Type: "websocket", + Host: types.ToString(wrapped.InternalEvent["host"]), + Matched: types.ToString(wrapped.InternalEvent["host"]), + Metadata: wrapped.OperatorsResult.PayloadValues, + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + Timestamp: time.Now(), + IP: types.ToString(wrapped.InternalEvent["ip"]), + Request: types.ToString(wrapped.InternalEvent["request"]), + Response: types.ToString(wrapped.InternalEvent["responses"]), + } + return data +} diff --git a/v2/pkg/protocols/others/wss/wss.go b/v2/pkg/protocols/others/wss/wss.go deleted file mode 100644 index c4004307..00000000 --- a/v2/pkg/protocols/others/wss/wss.go +++ /dev/null @@ -1 +0,0 @@ -package wss diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index bd711492..c6c2c24a 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -137,7 +137,8 @@ func (t *Template) Requests() int { len(t.RequestsNetwork) + len(t.RequestsHeadless) + len(t.Workflows) + - len(t.RequestsSSL) + len(t.RequestsSSL) + + len(t.RequestsWebsocket) return sum } @@ -175,4 +176,10 @@ func makeRequestsForTemplate(template *Template, options protocols.ExecuterOptio } template.Executer = executer.NewExecuter(requests, &options) } + if len(template.RequestsWebsocket) > 0 { + for _, req := range template.RequestsWebsocket { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, &options) + } } diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 6a19d2e5..ecd300fc 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/ssl" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/websocket" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) @@ -60,6 +61,9 @@ type Template struct { // description: | // SSL contains the SSL request to make in the template. RequestsSSL []*ssl.Request `yaml:"ssl,omitempty" json:"ssl,omitempty" jsonschema:"title=ssl requests to make,description=SSL requests to make for the template"` + // description: | + // Websocket contains the Websocket request to make in the template. + RequestsWebsocket []*websocket.Request `yaml:"websocket,omitempty" json:"websocket,omitempty" jsonschema:"title=websocket requests to make,description=Websocket requests to make for the template"` // description: | // Workflows is a yaml based workflow declaration code. From b76c1f99cc0898d41ecfe69a8ae7f94a522dab39 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 27 Sep 2021 20:58:05 +0530 Subject: [PATCH 006/146] Misc changes to ws protocol --- .../protocols/others/websocket/websocket.go | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/others/websocket/websocket.go b/v2/pkg/protocols/others/websocket/websocket.go index db62c663..4cf3ff74 100644 --- a/v2/pkg/protocols/others/websocket/websocket.go +++ b/v2/pkg/protocols/others/websocket/websocket.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "io" + "net/http" "net/url" "strings" "time" @@ -32,6 +33,9 @@ type Request struct { // description: | // Inputs contains inputs for the websocket protocol Inputs []*Input `yaml:"inputs,omitempty" jsonschema:"title=inputs for the websocket request,description=Inputs contains any input/output for the current request"` + // description: | + // Origin is the websocket request origin. + Origin string `yaml:"origin,omitempty" jsonschema:"title=origin is the request origin,description=Origin is the websocket request origin"` // description: | // Attack is the type of payload combinations to perform. @@ -163,7 +167,15 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + var header ws.HandshakeHeader + + if r.Origin != "" { + header = ws.HandshakeHeaderHTTP(http.Header{ + "Origin": []string{r.Origin}, + }) + } websocketDialer := ws.Dialer{ + Header: header, Timeout: time.Duration(r.options.Options.Timeout) * time.Second, NetDial: r.dialer.Dial, TLSConfig: &tls.Config{InsecureSkipVerify: true, ServerName: hostname}, @@ -179,7 +191,7 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu responseBuilder := &strings.Builder{} if readBuffer != nil { - io.Copy(responseBuilder, readBuffer) // Copy initial response + _, _ = io.Copy(responseBuilder, readBuffer) // Copy initial response } reqBuilder := &strings.Builder{} @@ -187,7 +199,6 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu inputEvents := make(map[string]interface{}) for _, req := range r.Inputs { reqBuilder.Grow(len(req.Data)) - reqBuilder.WriteString(req.Data) finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), dynamicValues) if dataErr != nil { @@ -195,6 +206,8 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu r.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } + reqBuilder.WriteString(string(finalData)) + err = wsutil.WriteClientMessage(conn, ws.OpText, finalData) if err != nil { r.options.Output.Request(r.options.TemplateID, input, "websocket", err) @@ -252,6 +265,7 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu for k, v := range inputEvents { data[k] = v } + data["success"] = "true" data["request"] = reqBuilder.String() data["response"] = responseBuilder.String() data["host"] = input @@ -294,7 +308,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out Timestamp: time.Now(), IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), - Response: types.ToString(wrapped.InternalEvent["responses"]), + Response: types.ToString(wrapped.InternalEvent["response"]), } return data } From 7405254c723806ba0bd254ca45adcc15b0fd8db3 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 29 Sep 2021 04:53:42 +0530 Subject: [PATCH 007/146] Misc work on ws protocol --- .../protocols/others/websocket/websocket.go | 47 +++++++++++++++---- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/v2/pkg/protocols/others/websocket/websocket.go b/v2/pkg/protocols/others/websocket/websocket.go index 4cf3ff74..c48677f8 100644 --- a/v2/pkg/protocols/others/websocket/websocket.go +++ b/v2/pkg/protocols/others/websocket/websocket.go @@ -30,12 +30,15 @@ type Request struct { operators.Operators `yaml:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-"` + // description: | + // Address contains address for the request + Address string `yaml:"address,omitempty" jsonschema:"title=address for the websocket request,description=Address contains address for the request"` // description: | // Inputs contains inputs for the websocket protocol Inputs []*Input `yaml:"inputs,omitempty" jsonschema:"title=inputs for the websocket request,description=Inputs contains any input/output for the current request"` // description: | - // Origin is the websocket request origin. - Origin string `yaml:"origin,omitempty" jsonschema:"title=origin is the request origin,description=Origin is the websocket request origin"` + // Headers contains headers for the request. + Headers map[string]string `yaml:"headers,omitempty" jsonschema:"title=headers contains the request headers,description=Headers contains headers for the request"` // description: | // Attack is the type of payload combinations to perform. @@ -167,21 +170,45 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - var header ws.HandshakeHeader + header := http.Header{} - if r.Origin != "" { - header = ws.HandshakeHeaderHTTP(http.Header{ - "Origin": []string{r.Origin}, - }) + payloadValues := make(map[string]interface{}) + for k, v := range dynamicValues { + payloadValues[k] = v + } + parsed, err := url.Parse(input) + if err != nil { + return errors.Wrap(err, "could not parse input url") + } + payloadValues["Address"] = parsed.Host + payloadValues["Scheme"] = parsed.Scheme + payloadValues["Path"] = parsed.Path + payloadValues["hostname"] = parsed.Hostname() + + for key, value := range r.Headers { + finalData, dataErr := expressions.EvaluateByte([]byte(value), payloadValues) + if dataErr != nil { + r.options.Output.Request(r.options.TemplateID, input, "websocket", dataErr) + r.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not evaluate template expressions") + } + header.Set(key, string(finalData)) } websocketDialer := ws.Dialer{ - Header: header, + Header: ws.HandshakeHeaderHTTP(header), Timeout: time.Duration(r.options.Options.Timeout) * time.Second, NetDial: r.dialer.Dial, TLSConfig: &tls.Config{InsecureSkipVerify: true, ServerName: hostname}, } - conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), input) + finalAddress, dataErr := expressions.EvaluateByte([]byte(r.Address), payloadValues) + if dataErr != nil { + r.options.Output.Request(r.options.TemplateID, input, "websocket", dataErr) + r.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not evaluate template expressions") + } + + conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), string(finalAddress)) if err != nil { r.options.Output.Request(r.options.TemplateID, input, "ssl", err) r.options.Progress.IncrementFailedRequestsBy(1) @@ -200,7 +227,7 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu for _, req := range r.Inputs { reqBuilder.Grow(len(req.Data)) - finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), dynamicValues) + finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues) if dataErr != nil { r.options.Output.Request(r.options.TemplateID, input, "websocket", dataErr) r.options.Progress.IncrementFailedRequestsBy(1) From f8d5d025702da2e9a477b0f04b2602cd2053abc2 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 12 Oct 2021 23:28:24 +0200 Subject: [PATCH 008/146] Adding support to read tcp data stream till the end --- v2/pkg/protocols/network/network.go | 7 +++++ v2/pkg/protocols/network/request.go | 40 ++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index b7de7b92..ef47dc36 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -60,6 +60,13 @@ type Request struct { // examples: // - value: "2048" ReadSize int `yaml:"read-size,omitempty" jsonschema:"title=size of network response to read,description=Size of response to read at the end. Default is 1024 bytes"` + // description: | + // ReadAll determines if the data stream should be read till the end regardless of the size + // + // Default value for read-all is false. + // examples: + // - value: false + ReadAll bool `yaml:"read-all,omitempty" jsonschema:"title=read all response stream,description=Read all response stream till the server stops sending"` // Operators for the current request go here. operators.Operators `yaml:",inline,omitempty"` diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index aad3c60e..ab517668 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -6,6 +6,7 @@ import ( "io" "net" "net/url" + "os" "strings" "time" @@ -183,13 +184,40 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin if r.ReadSize != 0 { bufferSize = r.ReadSize } - final := make([]byte, bufferSize) - n, err := conn.Read(final) - if err != nil && err != io.EOF { - r.options.Output.Request(r.options.TemplateID, address, "network", err) - return errors.Wrap(err, "could not read from server") + + var ( + final []byte + n int + ) + + if r.ReadAll { + readInterval := time.After(time.Second * 1) + read_socket: + for { + select { + case <-readInterval: + break read_socket + default: + buf := make([]byte, bufferSize) + nBuf, err := conn.Read(buf) + if err != nil && !os.IsTimeout(err) { + r.options.Output.Request(r.options.TemplateID, address, "network", err) + return errors.Wrap(err, "could not read from server") + } + responseBuilder.Write(buf[:nBuf]) + final = append(final, buf...) + n += nBuf + } + } + } else { + final = make([]byte, bufferSize) + n, err = conn.Read(final) + if err != nil && err != io.EOF { + r.options.Output.Request(r.options.TemplateID, address, "network", err) + return errors.Wrap(err, "could not read from server") + } + responseBuilder.Write(final[:n]) } - responseBuilder.Write(final[:n]) if r.options.Options.Debug || r.options.Options.DebugResponse { responseOutput := responseBuilder.String() From 14bb1b7b21a48d5bf012a3225367c252016cca03 Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Thu, 14 Oct 2021 23:30:37 +0200 Subject: [PATCH 009/146] Implement `-template-url` and `-workflow-url` for retrieving lists of templates/workflows to run. --- v2/cmd/nuclei/main.go | 2 + v2/internal/runner/runner.go | 2 + v2/pkg/catalog/loader/loader.go | 22 ++++-- v2/pkg/catalog/loader/remote_loader.go | 98 ++++++++++++++++++++++++++ v2/pkg/types/types.go | 4 ++ 5 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 v2/pkg/catalog/loader/remote_loader.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 83b13fc6..81e701fc 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -56,7 +56,9 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"), + flagSet.StringSliceVarP(&options.TemplateURLs, "template-urls", "tu", []string{}, "URL to a list of templates"), flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "list of workflows to run"), + flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-urls", "wu", []string{}, "URL to a list of workflows to run"), flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run newly added templates only"), flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"), diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index e87903f1..06abc70a 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -337,7 +337,9 @@ func (r *Runner) RunEnumeration() error { loaderConfig := loader.Config{ Templates: r.options.Templates, + TemplateURLs: r.options.TemplateURLs, Workflows: r.options.Workflows, + WorkflowURLs: r.options.WorkflowURLs, ExcludeTemplates: r.options.ExcludedTemplates, Tags: r.options.Tags, ExcludeTags: r.options.ExcludeTags, diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index eac22b4a..6d914ae5 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -15,7 +15,9 @@ import ( // Config contains the configuration options for the loader type Config struct { Templates []string + TemplateURLs []string Workflows []string + WorkflowURLs []string ExcludeTemplates []string IncludeTemplates []string @@ -37,6 +39,7 @@ type Store struct { pathFilter *filter.PathFilter config *Config finalTemplates []string + finalWorkflows []string templates []*templates.Template workflows []*templates.Template @@ -61,13 +64,24 @@ func New(config *Config) (*Store, error) { IncludedTemplates: config.IncludeTemplates, ExcludedTemplates: config.ExcludeTemplates, }, config.Catalog), + finalTemplates: config.Templates, + finalWorkflows: config.Workflows, + } + + if len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 { + remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs) + if err != nil { + return store, err + } + store.finalTemplates = append(store.finalTemplates, remoteTemplates...) + store.finalWorkflows = append(store.finalWorkflows, remoteWorkflows...) } // Handle a case with no templates or workflows, where we use base directory - if len(config.Templates) == 0 && len(config.Workflows) == 0 { - config.Templates = append(config.Templates, config.TemplatesDirectory) + if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 { + store.finalTemplates = []string{config.TemplatesDirectory} } - store.finalTemplates = append(store.finalTemplates, config.Templates...) + return store, nil } @@ -90,7 +104,7 @@ func (store *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) { // the complete compiled templates for a nuclei execution configuration. func (store *Store) Load() { store.templates = store.LoadTemplates(store.finalTemplates) - store.workflows = store.LoadWorkflows(store.config.Workflows) + store.workflows = store.LoadWorkflows(store.finalWorkflows) } // ValidateTemplates takes a list of templates and validates them diff --git a/v2/pkg/catalog/loader/remote_loader.go b/v2/pkg/catalog/loader/remote_loader.go new file mode 100644 index 00000000..ec47ed51 --- /dev/null +++ b/v2/pkg/catalog/loader/remote_loader.go @@ -0,0 +1,98 @@ +package loader + +import ( + "bufio" + "fmt" + "github.com/pkg/errors" + "net/http" + "strings" +) + +type ContentType string + +const ( + Template ContentType = "Template" + Workflow ContentType = "Workflow" +) + +type RemoteContentError struct { + Content []string + Type ContentType + Error error +} + +func getRemoteTemplatesAndWorkflows(templateURLs []string, workflowURLs []string) ([]string, []string, error) { + remoteContentErrorChannel := make(chan RemoteContentError) + + for _, templateURL := range templateURLs { + go getRemoteContent(templateURL, remoteContentErrorChannel, Template) + } + for _, workflowURL := range workflowURLs { + go getRemoteContent(workflowURL, remoteContentErrorChannel, Workflow) + } + + var remoteTemplateList []string + var remoteWorkFlowList []string + var err error + for i := 0; i < (len(templateURLs) + len(workflowURLs)); i++ { + remoteContentError := <-remoteContentErrorChannel + if remoteContentError.Error != nil { + if err != nil { + err = errors.New(remoteContentError.Error.Error() + ": " + err.Error()) + } else { + err = remoteContentError.Error + } + } else { + if remoteContentError.Type == Template { + remoteTemplateList = append(remoteTemplateList, remoteContentError.Content...) + } else if remoteContentError.Type == Workflow { + remoteWorkFlowList = append(remoteWorkFlowList, remoteContentError.Content...) + } + } + } + + return remoteTemplateList, remoteWorkFlowList, err +} + +func getRemoteContent(URL string, w chan<- RemoteContentError, contentType ContentType) { + response, err := http.Get(URL) + if err != nil { + w <- RemoteContentError{ + Error: err, + } + return + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode > 299 { + w <- RemoteContentError{ + Error: fmt.Errorf("get \"%s\": unexpect status %d", URL, response.StatusCode), + } + return + } + + if err != nil { + + } + + scanner := bufio.NewScanner(response.Body) + var templateList []string + for scanner.Scan() { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + templateList = append(templateList, text) + } + + if err := scanner.Err(); err != nil { + w <- RemoteContentError{ + Error: errors.Wrap(err, "get \"%s\""), + } + return + } + + w <- RemoteContentError{ + Content: templateList, + Type: contentType, + } +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index dd81c3e4..b5762a86 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -15,8 +15,12 @@ type Options struct { ExcludeTags goflags.NormalizedStringSlice // Workflows specifies any workflows to run by nuclei Workflows goflags.StringSlice + // WorkflowURLs specifies URLs to a list of workflows to use + WorkflowURLs goflags.StringSlice // Templates specifies the template/templates to use Templates goflags.StringSlice + // TemplateURLs specifies URLs to a list of templates to use + TemplateURLs goflags.StringSlice // ExcludedTemplates specifies the template/templates to exclude ExcludedTemplates goflags.StringSlice // CustomHeaders is the list of custom global headers to send with each request. From 81102750a1b384baefb195101be8c23e5f83c414 Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Thu, 14 Oct 2021 23:30:51 +0200 Subject: [PATCH 010/146] Implement integration tests for remote template and workflow urls. --- integration_tests/loader/basic.yaml | 10 ++ .../loader/condition-matched.yaml | 11 ++ integration_tests/loader/get-headers.yaml | 17 +++ integration_tests/loader/get.yaml | 15 +++ integration_tests/loader/template-list.yaml | 2 + integration_tests/loader/workflow-list.yaml | 2 + v2/cmd/integration-test/integration-test.go | 1 + v2/cmd/integration-test/loader.go | 116 ++++++++++++++++++ v2/internal/testutils/integration.go | 21 +++- 9 files changed, 190 insertions(+), 5 deletions(-) create mode 100644 integration_tests/loader/basic.yaml create mode 100644 integration_tests/loader/condition-matched.yaml create mode 100644 integration_tests/loader/get-headers.yaml create mode 100644 integration_tests/loader/get.yaml create mode 100644 integration_tests/loader/template-list.yaml create mode 100644 integration_tests/loader/workflow-list.yaml create mode 100644 v2/cmd/integration-test/loader.go diff --git a/integration_tests/loader/basic.yaml b/integration_tests/loader/basic.yaml new file mode 100644 index 00000000..f49193b1 --- /dev/null +++ b/integration_tests/loader/basic.yaml @@ -0,0 +1,10 @@ +id: workflow-example + +info: + name: Test Workflow Template + author: pdteam + severity: info + +workflows: + - template: workflow/match-1.yaml + - template: workflow/match-2.yaml \ No newline at end of file diff --git a/integration_tests/loader/condition-matched.yaml b/integration_tests/loader/condition-matched.yaml new file mode 100644 index 00000000..8b0a6573 --- /dev/null +++ b/integration_tests/loader/condition-matched.yaml @@ -0,0 +1,11 @@ +id: condition-matched-workflow + +info: + name: Condition Matched Workflow + author: pdteam + severity: info + +workflows: + - template: workflow/match-1.yaml + subtemplates: + - template: workflow/match-2.yaml \ No newline at end of file diff --git a/integration_tests/loader/get-headers.yaml b/integration_tests/loader/get-headers.yaml new file mode 100644 index 00000000..bae36705 --- /dev/null +++ b/integration_tests/loader/get-headers.yaml @@ -0,0 +1,17 @@ +id: basic-get-headers + +info: + name: Basic GET Headers Request + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + headers: + test: nuclei + matchers: + - type: word + words: + - "This is test headers matcher text" \ No newline at end of file diff --git a/integration_tests/loader/get.yaml b/integration_tests/loader/get.yaml new file mode 100644 index 00000000..c7e07e8c --- /dev/null +++ b/integration_tests/loader/get.yaml @@ -0,0 +1,15 @@ +id: basic-get + +info: + name: Basic GET Request + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + matchers: + - type: word + words: + - "This is test matcher text" \ No newline at end of file diff --git a/integration_tests/loader/template-list.yaml b/integration_tests/loader/template-list.yaml new file mode 100644 index 00000000..fae00d6c --- /dev/null +++ b/integration_tests/loader/template-list.yaml @@ -0,0 +1,2 @@ +loader/get.yaml +loader/get-headers.yaml diff --git a/integration_tests/loader/workflow-list.yaml b/integration_tests/loader/workflow-list.yaml new file mode 100644 index 00000000..3f56730e --- /dev/null +++ b/integration_tests/loader/workflow-list.yaml @@ -0,0 +1,2 @@ +loader/basic.yaml +loader/condition-matched.yaml diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index baa743e8..94e6c215 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -26,6 +26,7 @@ func main() { "network": networkTestcases, "dns": dnsTestCases, "workflow": workflowTestcases, + "loader": loaderTestcases, } for proto, tests := range protocolTests { if protocol == "" || protocol == proto { diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go new file mode 100644 index 00000000..13aa3f05 --- /dev/null +++ b/v2/cmd/integration-test/loader.go @@ -0,0 +1,116 @@ +package main + +import ( + "fmt" + "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "net/http" + "net/http/httptest" + "os" + "strings" +) + +var loaderTestcases = map[string]testutils.TestCase{ + "loader/template-list.yaml": &remoteTemplateList{}, + "loader/workflow-list.yaml": &remoteWorkflowList{}, + "loader/nonexistent-template-list.yaml": &nonExistentTemplateList{}, + "loader/nonexistent-workflow-list.yaml": &nonExistentWorkflowList{}, +} + +type remoteTemplateList struct{} + +// Execute executes a test case and returns an error if occurred +func (h *remoteTemplateList) Execute(templateList string) error { + router := httprouter.New() + + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "This is test matcher text") + if strings.EqualFold(r.Header.Get("test"), "nuclei") { + fmt.Fprintf(w, "This is test headers matcher text") + } + })) + + router.GET("/template_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + file, err := os.ReadFile(templateList) + if err != nil { + w.WriteHeader(500) + } + w.Write(file) + })) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list") + if err != nil { + return err + } + if len(results) != 2 { + return errIncorrectResultsCount(results) + } + return nil +} + +type remoteWorkflowList struct{} + +// Execute executes a test case and returns an error if occurred +func (h *remoteWorkflowList) Execute(workflowList string) error { + router := httprouter.New() + + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "This is test matcher text") + if strings.EqualFold(r.Header.Get("test"), "nuclei") { + fmt.Fprintf(w, "This is test headers matcher text") + } + })) + + router.GET("/workflow_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + file, err := os.ReadFile(workflowList) + if err != nil { + w.WriteHeader(500) + } + w.Write(file) + })) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list") + if err != nil { + return err + } + if len(results) != 3 { + return errIncorrectResultsCount(results) + } + return nil +} + +type nonExistentTemplateList struct{} + +// Execute executes a test case and returns an error if occurred +func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error { + router := httprouter.New() + ts := httptest.NewServer(router) + defer ts.Close() + + _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/404") + if err == nil { + return fmt.Errorf("expected error for nonexisting workflow url") + } + + return nil +} + +type nonExistentWorkflowList struct{} + +// Execute executes a test case and returns an error if occurred +func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error { + router := httprouter.New() + ts := httptest.NewServer(router) + defer ts.Close() + + _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/404") + if err == nil { + return fmt.Errorf("expected error for nonexisting workflow url") + } + + return nil +} diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index f2c39886..4435b957 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -12,15 +12,15 @@ import ( // RunNucleiTemplateAndGetResults returns a list of results for a template func RunNucleiTemplateAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { - return runNucleiAndGetResults(true, template, url, debug, extra...) + return RunNucleiAndGetResults(true, template, url, debug, extra...) } // RunNucleiWorkflowAndGetResults returns a list of results for a workflow func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { - return runNucleiAndGetResults(false, template, url, debug, extra...) + return RunNucleiAndGetResults(false, template, url, debug, extra...) } -func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) { +func RunNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) { var templateOrWorkflowFlag string if isTemplate { templateOrWorkflowFlag = "-t" @@ -28,11 +28,22 @@ func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, e templateOrWorkflowFlag = "-w" } - cmd := exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-silent") + return RunNucleiBareArgsAndGetResults(debug, append([]string{ + templateOrWorkflowFlag, + template, + "-target", + url, + }, extra...)...) +} + +func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, error) { + cmd := exec.Command("./nuclei") if debug { - cmd = exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-debug") + cmd.Args = append(cmd.Args, "-debug") cmd.Stderr = os.Stderr fmt.Println(cmd.String()) + } else { + cmd.Args = append(cmd.Args, "-silent") } cmd.Args = append(cmd.Args, extra...) data, err := cmd.Output() From 5584fc285d6377bc0fb53bf14385a1f7b09bef10 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Tue, 19 Oct 2021 00:58:49 +0300 Subject: [PATCH 011/146] Add regexp pattern for template-id --- v2/pkg/templates/templates.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index b6f2491a..0b925d7e 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -27,7 +27,7 @@ type Template struct { // examples: // - name: ID Example // value: "\"CVE-2021-19520\"" - ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520"` + ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^(?:[a-zA-Z0-9][a-zA-Z0-9]*-?[a-zA-Z0-9]+?)+$"` // description: | // Info contains metadata information about the template. // examples: From 0ba3b19f1f7653a41327315644b7fcccb57de1c2 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 19 Oct 2021 11:31:36 +0200 Subject: [PATCH 012/146] fixing merge issues --- v2/pkg/protocols/network/request.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 399699a0..a1425eba 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -191,13 +191,13 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input if request.ReadSize != 0 { bufferSize = request.ReadSize } - - var ( + + var ( final []byte n int ) - if r.ReadAll { + if request.ReadAll { readInterval := time.After(time.Second * 1) read_socket: for { @@ -208,7 +208,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input buf := make([]byte, bufferSize) nBuf, err := conn.Read(buf) if err != nil && !os.IsTimeout(err) { - r.options.Output.Request(r.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplateID, address, "network", err) return errors.Wrap(err, "could not read from server") } responseBuilder.Write(buf[:nBuf]) From ae99a88f52b4ddb260818a8ba80a28f650233894 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 19 Oct 2021 19:39:33 +0200 Subject: [PATCH 013/146] timer.after => timer --- v2/pkg/protocols/network/request.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index a1425eba..6beecc6e 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -198,17 +198,25 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input ) if request.ReadAll { - readInterval := time.After(time.Second * 1) + readInterval := time.NewTimer(time.Second * 1) + // stop the timer and drain the channel + closeTimer := func(t *time.Timer) { + if !t.Stop() { + <-t.C + } + } read_socket: for { select { - case <-readInterval: + case <-readInterval.C: + closeTimer(readInterval) break read_socket default: buf := make([]byte, bufferSize) nBuf, err := conn.Read(buf) if err != nil && !os.IsTimeout(err) { request.options.Output.Request(request.options.TemplateID, address, "network", err) + closeTimer(readInterval) return errors.Wrap(err, "could not read from server") } responseBuilder.Write(buf[:nBuf]) From a1e099493dc3448465eccb7ee3072487fd03403f Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Wed, 20 Oct 2021 00:43:33 +0300 Subject: [PATCH 014/146] Use simpler regexp for template id validation --- v2/pkg/templates/templates.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 0b925d7e..a9a0e14e 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -27,7 +27,7 @@ type Template struct { // examples: // - name: ID Example // value: "\"CVE-2021-19520\"" - ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^(?:[a-zA-Z0-9][a-zA-Z0-9]*-?[a-zA-Z0-9]+?)+$"` + ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^[A-Za-z0-9-_]+$"` // description: | // Info contains metadata information about the template. // examples: From cbc5192e9a3e1d7f5409385ef26885d0154c1c6e Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 20 Oct 2021 00:02:06 +0200 Subject: [PATCH 015/146] Adding proxy socks support to headless browser --- .../protocols/headless/engine/http_client.go | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index b4747d54..9da2aaa9 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -1,13 +1,17 @@ package engine import ( + "context" "crypto/tls" + "fmt" + "net" "net/http" "net/url" "time" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/types" + "golang.org/x/net/proxy" ) // newhttpClient creates a new http client for headless communication with a timeout @@ -28,6 +32,22 @@ func newhttpClient(options *types.Options) *http.Client { if proxyURL, err := url.Parse(options.ProxyURL); err == nil { transport.Proxy = http.ProxyURL(proxyURL) } + } else if options.ProxySocksURL != "" { + var proxyAuth *proxy.Auth + + socksURL, proxyErr := url.Parse(options.ProxySocksURL) + if proxyErr == nil { + proxyAuth = &proxy.Auth{} + proxyAuth.User = socksURL.User.Username() + proxyAuth.Password, _ = socksURL.User.Password() + } + dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) + dc := dialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + if proxyErr == nil { + transport.DialContext = dc.DialContext + } } return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second} From 9e0144b6aede383452969ae81a771d231dfdd6e2 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Wed, 20 Oct 2021 01:31:38 +0300 Subject: [PATCH 016/146] Add template-id validation --- v2/pkg/parsers/parser.go | 30 ++++++++++------ v2/pkg/parsers/parser_test.go | 68 +++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 11 deletions(-) create mode 100644 v2/pkg/parsers/parser_test.go diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index b65cb79e..1d5213ae 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -17,7 +17,10 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" ) -const mandatoryFieldMissingTemplate = "mandatory '%s' field is missing" +const ( + mandatoryFieldMissingTemplate = "mandatory '%s' field is missing" + invalidFieldFormatTemplate = "invalid field format for '%s' (allowed format is %s)" +) // LoadTemplate returns true if the template is valid and matches the filtering criteria. func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string) (bool, error) { @@ -30,12 +33,11 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags [] return false, nil } - templateInfo := template.Info - if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil { + if validationError := validateTemplateFields(template); validationError != nil { return false, validationError } - return isTemplateInfoMetadataMatch(tagFilter, &templateInfo, extraTags) + return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags) } // LoadWorkflow returns true if the workflow is valid and matches the filtering criteria. @@ -45,10 +47,8 @@ func LoadWorkflow(templatePath string) (bool, error) { return false, templateParseError } - templateInfo := template.Info - if len(template.Workflows) > 0 { - if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil { + if validationError := validateTemplateFields(template); validationError != nil { return false, validationError } return true, nil @@ -71,10 +71,8 @@ func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *mode return match, err } -func validateMandatoryInfoFields(info *model.Info) error { - if info == nil { - return fmt.Errorf(mandatoryFieldMissingTemplate, "info") - } +func validateTemplateFields(template *templates.Template) error { + info := template.Info if utils.IsBlank(info.Name) { return fmt.Errorf(mandatoryFieldMissingTemplate, "name") @@ -83,6 +81,15 @@ func validateMandatoryInfoFields(info *model.Info) error { if info.Authors.IsEmpty() { return fmt.Errorf(mandatoryFieldMissingTemplate, "author") } + + if template.ID == "" { + return fmt.Errorf(mandatoryFieldMissingTemplate, "id") + } + + if !templateIDRegexp.MatchString(template.ID) { + return fmt.Errorf(invalidFieldFormatTemplate, "id", templateIDRegexp.String()) + } + return nil } @@ -90,6 +97,7 @@ var ( parsedTemplatesCache *cache.Templates ShouldValidate bool fieldErrorRegexp = regexp.MustCompile(`not found in`) + templateIDRegexp = regexp.MustCompile(`^[A-Za-z0-9-_]+$`) ) const ( diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go new file mode 100644 index 00000000..e1e2a128 --- /dev/null +++ b/v2/pkg/parsers/parser_test.go @@ -0,0 +1,68 @@ +package parsers + +import ( + "errors" + "testing" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" + "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/stretchr/testify/require" +) + +func TestLoadTemplate(t *testing.T) { + origTemplatesCache := parsedTemplatesCache + defer func() { parsedTemplatesCache = origTemplatesCache }() + + tt := []struct { + name string + template *templates.Template + templateErr error + + expectedErr error + }{ + { + name: "valid", + template: &templates.Template{ + ID: "CVE-2021-27330", + Info: model.Info{ + Name: "Valid template", + Authors: stringslice.StringSlice{Value: "Author"}, + }, + }, + }, + { + name: "missingName", + template: &templates.Template{}, + expectedErr: errors.New("mandatory 'name' field is missing"), + }, + { + name: "invalidID", + template: &templates.Template{ + ID: "ABC DEF", + Info: model.Info{ + Name: "Invalid ID", + Authors: stringslice.StringSlice{Value: "Author"}, + }, + }, + expectedErr: errors.New("invalid field format for 'id' (allowed format is ^[A-Za-z0-9-_]+$)"), + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + parsedTemplatesCache.Store(tc.name, tc.template, tc.templateErr) + + tagFilter := filter.New(&filter.Config{}) + success, err := LoadTemplate(tc.name, tagFilter, nil) + if tc.expectedErr == nil { + require.NoError(t, err) + require.True(t, success) + } else { + require.Equal(t, tc.expectedErr, err) + require.False(t, success) + } + }) + } +} From df12b0d2e6ac754aa5691addf0c327af2a6ae0b3 Mon Sep 17 00:00:00 2001 From: kchason Date: Tue, 19 Oct 2021 23:06:02 -0400 Subject: [PATCH 017/146] Add CLI options for passing client cert auth --- v2/cmd/nuclei/main.go | 3 +++ v2/pkg/types/types.go | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 23fd7b1d..c474615a 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -105,6 +105,9 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"), flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"), flagSet.BoolVar(&options.EnvironmentVariables, "env-vars", false, "enable environment variables support"), + flagSet.StringVar(&options.ClientCertFile, "client-cert", "cc", "client certificate file (PEM-encoded) used for authenticating against scanned hosts"), + flagSet.StringVar(&options.ClientKeyFile, "client-key", "ck", "client key file (PEM-encoded) used for authenticating against scanned hosts"), + flagSet.StringVar(&options.ClientCAFile, "client-ca", "ca", "client certificate authority file (PEM-encoded) used for authenticating against scanned hosts"), ) createGroup(flagSet, "interactsh", "interactsh", diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index dafb2f3e..16bb8169 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -156,4 +156,10 @@ type Options struct { NoUpdateTemplates bool // EnvironmentVariables enables support for environment variables EnvironmentVariables bool + // ClientCertFile client certificate file (PEM-encoded) used for authenticating against scanned hosts + ClientCertFile string + // ClientKeyFile client key file (PEM-encoded) used for authenticating against scanned hosts + ClientKeyFile string + // ClientCAFile client certificate authority file (PEM-encoded) used for authenticating against scanned hosts + ClientCAFile string } From e6728e8ff9f4d1d25d91b0f9d047dbca84010308 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 20 Oct 2021 13:26:47 +0200 Subject: [PATCH 018/146] Making headless httpclient more similar to real browsers --- v2/pkg/protocols/headless/engine/http_client.go | 15 ++++++++++++++- v2/pkg/protocols/headless/engine/rules.go | 3 +++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index b4747d54..94c33563 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -3,6 +3,7 @@ package engine import ( "crypto/tls" "net/http" + "net/http/cookiejar" "net/url" "time" @@ -30,5 +31,17 @@ func newhttpClient(options *types.Options) *http.Client { } } - return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second} + jar, _ := cookiejar.New(nil) + + httpclient := &http.Client{ + Transport: transport, + Timeout: time.Duration(options.Timeout*3) * time.Second, + Jar: jar, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + // the browser should follow redirects not us + return http.ErrUseLastResponse + }, + } + + return httpclient } diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index a254902e..a802a64a 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -8,6 +8,9 @@ import ( // routingRuleHandler handles proxy rule for actions related to request/response modification func (p *Page) routingRuleHandler(ctx *rod.Hijack) { + // usually browsers don't use chunked transfer encoding so we set the content-length nevertheless + ctx.Request.Req().ContentLength = int64(len(ctx.Request.Body())) + for _, rule := range p.rules { if rule.Part != "request" { continue From e2052dedc1f9abd0316ed9257bdd92f9e7e56dac Mon Sep 17 00:00:00 2001 From: kchason Date: Wed, 20 Oct 2021 09:44:14 -0400 Subject: [PATCH 019/146] Add options to readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 17d7d211..823f3a21 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,9 @@ CONFIGURATIONS: -system-resolvers use system DNS resolving as error fallback -passive enable passive HTTP response processing mode -env-vars enable environment variables support + -cc -client-cert client certificate file (PEM-encoded) used for authenticating against scanned hosts + -ck -client-key client key file (PEM-encoded) used for authenticating against scanned hosts + -ca -client-ca client certificate authority file (PEM-encoded) used for authenticating against scanned hosts INTERACTSH: -no-interactsh disable interactsh server for OOB testing From 9c77f15012d87d9126fb44096e935c309e998ad4 Mon Sep 17 00:00:00 2001 From: kchason Date: Wed, 20 Oct 2021 11:32:09 -0400 Subject: [PATCH 020/146] Argument checks for presence and validity --- v2/cmd/nuclei/main.go | 6 +++--- v2/internal/runner/options.go | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index c474615a..c4e76e2b 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -105,9 +105,9 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"), flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"), flagSet.BoolVar(&options.EnvironmentVariables, "env-vars", false, "enable environment variables support"), - flagSet.StringVar(&options.ClientCertFile, "client-cert", "cc", "client certificate file (PEM-encoded) used for authenticating against scanned hosts"), - flagSet.StringVar(&options.ClientKeyFile, "client-key", "ck", "client key file (PEM-encoded) used for authenticating against scanned hosts"), - flagSet.StringVar(&options.ClientCAFile, "client-ca", "ca", "client certificate authority file (PEM-encoded) used for authenticating against scanned hosts"), + flagSet.StringVarP(&options.ClientCertFile, "client-cert", "cc", "", "client certificate file (PEM-encoded) used for authenticating against scanned hosts"), + flagSet.StringVarP(&options.ClientKeyFile, "client-key", "ck", "", "client key file (PEM-encoded) used for authenticating against scanned hosts"), + flagSet.StringVarP(&options.ClientCAFile, "client-ca", "ca", "", "client certificate authority file (PEM-encoded) used for authenticating against scanned hosts"), ) createGroup(flagSet, "interactsh", "interactsh", diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index cba1af5b..fd43722b 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -98,6 +98,14 @@ func validateOptions(options *types.Options) error { validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) } + // Verify if any of the client certificate options were set since it requires all three to work properly + if len(options.ClientCertFile) > 0 || len(options.ClientKeyFile) > 0 || len(options.ClientCAFile) > 0 { + if len(options.ClientCertFile) == 0 || len(options.ClientKeyFile) == 0 || len(options.ClientCAFile) == 0 { + return errors.New("if a client certification option is provided, then all three must be provided") + } + validateCertificatePaths([]string{options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile}) + } + return nil } @@ -174,3 +182,14 @@ func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPat } } } + +func validateCertificatePaths(certificatePaths []string) { + for _, certificatePath := range certificatePaths { + if _, err := os.Stat(certificatePath); os.IsNotExist(err) { + // The provided path to the PEM certificate does not exist for the client authentication. As this is + // required for successful authentication, log and return an error + gologger.Fatal().Msgf("The given path (%s) to the certificate does not exist!", certificatePath) + break + } + } +} From f1cd0a5d288de0fc4212287ec1f0a06033e9fc7b Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Wed, 20 Oct 2021 22:58:36 +0300 Subject: [PATCH 021/146] Update template id regexp --- v2/pkg/parsers/parser.go | 2 +- v2/pkg/parsers/parser_test.go | 2 +- v2/pkg/templates/templates.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 1d5213ae..7bde724f 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -97,7 +97,7 @@ var ( parsedTemplatesCache *cache.Templates ShouldValidate bool fieldErrorRegexp = regexp.MustCompile(`not found in`) - templateIDRegexp = regexp.MustCompile(`^[A-Za-z0-9-_]+$`) + templateIDRegexp = regexp.MustCompile(`^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$`) ) const ( diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go index e1e2a128..fefde323 100644 --- a/v2/pkg/parsers/parser_test.go +++ b/v2/pkg/parsers/parser_test.go @@ -46,7 +46,7 @@ func TestLoadTemplate(t *testing.T) { Authors: stringslice.StringSlice{Value: "Author"}, }, }, - expectedErr: errors.New("invalid field format for 'id' (allowed format is ^[A-Za-z0-9-_]+$)"), + expectedErr: errors.New("invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"), }, } diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index a9a0e14e..8f5a4e16 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -27,7 +27,7 @@ type Template struct { // examples: // - name: ID Example // value: "\"CVE-2021-19520\"" - ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^[A-Za-z0-9-_]+$"` + ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"` // description: | // Info contains metadata information about the template. // examples: From 5d0f6b2622f54aa8f1980f5ef83138523e23c26a Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Wed, 20 Oct 2021 23:14:04 +0300 Subject: [PATCH 022/146] Improve invalid template id tests --- v2/pkg/parsers/parser_test.go | 54 ++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go index fefde323..e94b2ee3 100644 --- a/v2/pkg/parsers/parser_test.go +++ b/v2/pkg/parsers/parser_test.go @@ -2,6 +2,7 @@ package parsers import ( "errors" + "fmt" "testing" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" @@ -37,17 +38,6 @@ func TestLoadTemplate(t *testing.T) { template: &templates.Template{}, expectedErr: errors.New("mandatory 'name' field is missing"), }, - { - name: "invalidID", - template: &templates.Template{ - ID: "ABC DEF", - Info: model.Info{ - Name: "Invalid ID", - Authors: stringslice.StringSlice{Value: "Author"}, - }, - }, - expectedErr: errors.New("invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"), - }, } for _, tc := range tt { @@ -65,4 +55,46 @@ func TestLoadTemplate(t *testing.T) { } }) } + + t.Run("invalidTemplateID", func(t *testing.T) { + tt := []struct { + id string + success bool + }{ + {id: "A-B-C", success: true}, + {id: "A-B-C-1", success: true}, + {id: "CVE_2021_27330", success: true}, + {id: "ABC DEF", success: false}, + {id: "_-__AAA_", success: false}, + {id: " CVE-2021-27330", success: false}, + {id: "CVE-2021-27330 ", success: false}, + {id: "CVE-2021-27330-", success: false}, + {id: "-CVE-2021-27330-", success: false}, + {id: "CVE-2021--27330", success: false}, + {id: "CVE-2021+27330", success: false}, + } + for i, tc := range tt { + name := fmt.Sprintf("regexp%d", i) + t.Run(name, func(t *testing.T) { + template := &templates.Template{ + ID: tc.id, + Info: model.Info{ + Name: "Valid template", + Authors: stringslice.StringSlice{Value: "Author"}, + }, + } + parsedTemplatesCache.Store(name, template, nil) + + tagFilter := filter.New(&filter.Config{}) + success, err := LoadTemplate(name, tagFilter, nil) + if tc.success { + require.NoError(t, err) + require.True(t, success) + } else { + require.Equal(t, errors.New("invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"), err) + require.False(t, success) + } + }) + } + }) } From ff7a5997a225a81ed35b9017b76105ffe186111f Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Wed, 20 Oct 2021 23:24:11 +0300 Subject: [PATCH 023/146] Return multiple errors in template validation --- v2/pkg/parsers/parser.go | 15 ++++++++++----- v2/pkg/parsers/parser_test.go | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 7bde724f..83059baa 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "regexp" + "strings" "gopkg.in/yaml.v2" @@ -74,20 +75,24 @@ func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *mode func validateTemplateFields(template *templates.Template) error { info := template.Info + var errors []string + if utils.IsBlank(info.Name) { - return fmt.Errorf(mandatoryFieldMissingTemplate, "name") + errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "name")) } if info.Authors.IsEmpty() { - return fmt.Errorf(mandatoryFieldMissingTemplate, "author") + errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "author")) } if template.ID == "" { - return fmt.Errorf(mandatoryFieldMissingTemplate, "id") + errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "id")) + } else if !templateIDRegexp.MatchString(template.ID) { + errors = append(errors, fmt.Sprintf(invalidFieldFormatTemplate, "id", templateIDRegexp.String())) } - if !templateIDRegexp.MatchString(template.ID) { - return fmt.Errorf(invalidFieldFormatTemplate, "id", templateIDRegexp.String()) + if len(errors) > 0 { + return fmt.Errorf(strings.Join(errors, ", ")) } return nil diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go index e94b2ee3..ef74a317 100644 --- a/v2/pkg/parsers/parser_test.go +++ b/v2/pkg/parsers/parser_test.go @@ -34,9 +34,19 @@ func TestLoadTemplate(t *testing.T) { }, }, { - name: "missingName", + name: "emptyTemplate", template: &templates.Template{}, - expectedErr: errors.New("mandatory 'name' field is missing"), + expectedErr: errors.New("mandatory 'name' field is missing, mandatory 'author' field is missing, mandatory 'id' field is missing"), + }, + { + name: "emptyNameWithInvalidID", + template: &templates.Template{ + ID: "invalid id", + Info: model.Info{ + Authors: stringslice.StringSlice{Value: "Author"}, + }, + }, + expectedErr: errors.New("mandatory 'name' field is missing, invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"), }, } From fdb7c0d6cda8b2acf11af247d9eaf65f5b58f036 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 21 Oct 2021 06:20:40 +0530 Subject: [PATCH 024/146] dev update --- v2/pkg/catalog/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 8a996b62..91b153cf 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -26,7 +26,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.5.3` +const Version = `2.5.4-dev` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() From 450af73eee9e6f2a8444fb42e8f6983e6453e312 Mon Sep 17 00:00:00 2001 From: kchason Date: Thu, 21 Oct 2021 13:44:51 -0400 Subject: [PATCH 025/146] Fix README spacing with commas --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 823f3a21..37e2b745 100644 --- a/README.md +++ b/README.md @@ -128,9 +128,9 @@ CONFIGURATIONS: -system-resolvers use system DNS resolving as error fallback -passive enable passive HTTP response processing mode -env-vars enable environment variables support - -cc -client-cert client certificate file (PEM-encoded) used for authenticating against scanned hosts - -ck -client-key client key file (PEM-encoded) used for authenticating against scanned hosts - -ca -client-ca client certificate authority file (PEM-encoded) used for authenticating against scanned hosts + -cc, -client-cert client certificate file (PEM-encoded) used for authenticating against scanned hosts + -ck, -client-key client key file (PEM-encoded) used for authenticating against scanned hosts + -ca, -client-ca client certificate authority file (PEM-encoded) used for authenticating against scanned hosts INTERACTSH: -no-interactsh disable interactsh server for OOB testing From 0a57a1aa4d01095bd936fe3d64df3b73c19351fb Mon Sep 17 00:00:00 2001 From: kchason Date: Thu, 21 Oct 2021 13:48:13 -0400 Subject: [PATCH 026/146] Client certificate authentication for headless connections --- .../protocols/headless/engine/http_client.go | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index c9ec6e0c..eed7875a 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -2,6 +2,9 @@ package engine import ( "crypto/tls" + "crypto/x509" + "io/ioutil" + "log" "net/http" "time" @@ -12,15 +15,40 @@ import ( // newhttpClient creates a new http client for headless communication with a timeout func newhttpClient(options *types.Options) *http.Client { dialer := protocolstate.Dialer + + // Set the base TLS configuration definition + tlsConfig := &tls.Config{ + Renegotiation: tls.RenegotiateOnceAsClient, + InsecureSkipVerify: true, + } + + // Build the TLS config with the client certificate if it has been configured with the appropriate options. + // Only one of the options needs to be checked since the validation checks in main.go ensure that all three + // files are set if any of the client certification configuration options are. + if len(options.ClientCertFile) > 0 { + // Load the client certificate using the PEM encoded client certificate and the private key file + cert, err := tls.LoadX509KeyPair(options.ClientCertFile, options.ClientKeyFile) + if err != nil { + log.Fatal(err) + } + tlsConfig.Certificates = []tls.Certificate{cert} + + // Load the certificate authority PEM certificate into the TLS configuration + caCert, err := ioutil.ReadFile(options.ClientCAFile) + if err != nil { + log.Fatal(err) + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + tlsConfig.RootCAs = caCertPool + } + transport := &http.Transport{ DialContext: dialer.Dial, MaxIdleConns: 500, MaxIdleConnsPerHost: 500, MaxConnsPerHost: 500, - TLSClientConfig: &tls.Config{ - Renegotiation: tls.RenegotiateOnceAsClient, - InsecureSkipVerify: true, - }, + TLSClientConfig: tlsConfig, } return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second} } From f5b9eb32a1c5a3727fe21d47ac1af5083207afb2 Mon Sep 17 00:00:00 2001 From: kchason Date: Thu, 21 Oct 2021 13:54:56 -0400 Subject: [PATCH 027/146] Client certificate authentication for pooled connections --- .../http/httpclientpool/clientpool.go | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index a4faef07..3b0ce9b4 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -3,7 +3,10 @@ package httpclientpool import ( "context" "crypto/tls" + "crypto/x509" "fmt" + "io/ioutil" + "log" "net" "net/http" "net/http/cookiejar" @@ -161,16 +164,40 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl disableKeepAlives = configuration.Connection.DisableKeepAlive } + // Set the base TLS configuration definition + tlsConfig := &tls.Config{ + Renegotiation: tls.RenegotiateOnceAsClient, + InsecureSkipVerify: true, + } + + // Build the TLS config with the client certificate if it has been configured with the appropriate options. + // Only one of the options needs to be checked since the validation checks in main.go ensure that all three + // files are set if any of the client certification configuration options are. + if len(options.ClientCertFile) > 0 { + // Load the client certificate using the PEM encoded client certificate and the private key file + cert, err := tls.LoadX509KeyPair(options.ClientCertFile, options.ClientKeyFile) + if err != nil { + log.Fatal(err) + } + tlsConfig.Certificates = []tls.Certificate{cert} + + // Load the certificate authority PEM certificate into the TLS configuration + caCert, err := ioutil.ReadFile(options.ClientCAFile) + if err != nil { + log.Fatal(err) + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + tlsConfig.RootCAs = caCertPool + } + transport := &http.Transport{ DialContext: Dialer.Dial, MaxIdleConns: maxIdleConns, MaxIdleConnsPerHost: maxIdleConnsPerHost, MaxConnsPerHost: maxConnsPerHost, - TLSClientConfig: &tls.Config{ - Renegotiation: tls.RenegotiateOnceAsClient, - InsecureSkipVerify: true, - }, - DisableKeepAlives: disableKeepAlives, + TLSClientConfig: tlsConfig, + DisableKeepAlives: disableKeepAlives, } // Attempts to overwrite the dial function with the socks proxied version From 18deddb07abcabc056c81f19a4149f0de0449807 Mon Sep 17 00:00:00 2001 From: kchason Date: Thu, 21 Oct 2021 14:10:47 -0400 Subject: [PATCH 028/146] Missing error package --- v2/pkg/protocols/http/request.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 1a619c17..44793cdc 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -2,6 +2,7 @@ package http import ( "bytes" + "errors" "fmt" "io" "io/ioutil" From 99492911a36adb869e4ab17e5426084250b5f75a Mon Sep 17 00:00:00 2001 From: kchason Date: Thu, 21 Oct 2021 14:21:52 -0400 Subject: [PATCH 029/146] Revert "Missing error package" This reverts commit 18deddb07abcabc056c81f19a4149f0de0449807. --- v2/pkg/protocols/http/request.go | 1 - 1 file changed, 1 deletion(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 44793cdc..1a619c17 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -2,7 +2,6 @@ package http import ( "bytes" - "errors" "fmt" "io" "io/ioutil" From a911245d26a11f6a3f9aef61f6e182cbfe131d7f Mon Sep 17 00:00:00 2001 From: kchason Date: Sun, 24 Oct 2021 22:53:28 -0400 Subject: [PATCH 030/146] Re-add short command for env-vars lost during merge conflict resolution --- README.md | 2 +- v2/cmd/nuclei/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 757264fb..ca198881 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ CONFIGURATIONS: -r, -resolvers string file containing resolver list for nuclei -sr, -system-resolvers use system DNS resolving as error fallback -passive enable passive HTTP response processing mode - -env-vars enable environment variables support to be used in template + -ev, -env-vars enable environment variables support to be used in template -cc, -client-cert client certificate file (PEM-encoded) used for authenticating against scanned hosts -ck, -client-key client key file (PEM-encoded) used for authenticating against scanned hosts -ca, -client-ca client certificate authority file (PEM-encoded) used for authenticating against scanned hosts diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 43b9608c..1b898f68 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -92,7 +92,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"), flagSet.BoolVarP(&options.SystemResolvers, "system-resolvers", "sr", false, "use system DNS resolving as error fallback"), flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"), - flagSet.BoolVar(&options.EnvironmentVariables, "env-vars", false, "enable environment variables to be used in template"), + flagSet.BoolVarP(&options.EnvironmentVariables, "env-vars", "ev", false, "enable environment variables to be used in template"), flagSet.StringVarP(&options.ClientCertFile, "client-cert", "cc", "", "client certificate file (PEM-encoded) used for authenticating against scanned hosts"), flagSet.StringVarP(&options.ClientKeyFile, "client-key", "ck", "", "client key file (PEM-encoded) used for authenticating against scanned hosts"), flagSet.StringVarP(&options.ClientCAFile, "client-ca", "ca", "", "client certificate authority file (PEM-encoded) used for authenticating against scanned hosts"), From 1f8a9474cfa07682dc7e38a897b5b938a10a83f0 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 25 Oct 2021 12:00:09 +0000 Subject: [PATCH 031/146] Auto Generate Syntax Docs + JSONSchema [Mon Oct 25 12:00:09 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 25 ++++++++++++++++++++++ nuclei-jsonschema.json | 5 +++++ v2/pkg/templates/templates_doc.go | 35 ++++++++++++++++++------------- 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 30ed998f..a3fffe35 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -2392,6 +2392,31 @@ read-size: 2048 ``` + + +
+ +
+ +read-all bool + +
+
+ +ReadAll determines if the data stream should be read till the end regardless of the size + +Default value for read-all is false. + + + +Examples: + + +```yaml +read-all: false +``` + +
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 56253b7e..57f8fc7f 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -809,6 +809,11 @@ "title": "size of network response to read", "description": "Size of response to read at the end. Default is 1024 bytes" }, + "read-all": { + "type": "boolean", + "title": "read all response stream", + "description": "Read all response stream till the server stops sending" + }, "matchers": { "items": { "$ref": "#/definitions/matchers.Matcher" diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 0b19e675..d6042dfb 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -843,7 +843,7 @@ func init() { FieldName: "network", }, } - NETWORKRequestDoc.Fields = make([]encoder.Doc, 9) + NETWORKRequestDoc.Fields = make([]encoder.Doc, 10) NETWORKRequestDoc.Fields[0].Name = "id" NETWORKRequestDoc.Fields[0].Type = "string" NETWORKRequestDoc.Fields[0].Note = "" @@ -883,22 +883,29 @@ func init() { NETWORKRequestDoc.Fields[5].Comments[encoder.LineComment] = "ReadSize is the size of response to read at the end" NETWORKRequestDoc.Fields[5].AddExample("", 2048) - NETWORKRequestDoc.Fields[6].Name = "matchers" - NETWORKRequestDoc.Fields[6].Type = "[]matchers.Matcher" + NETWORKRequestDoc.Fields[6].Name = "read-all" + NETWORKRequestDoc.Fields[6].Type = "bool" NETWORKRequestDoc.Fields[6].Note = "" - NETWORKRequestDoc.Fields[6].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument." - NETWORKRequestDoc.Fields[6].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify" - NETWORKRequestDoc.Fields[7].Name = "extractors" - NETWORKRequestDoc.Fields[7].Type = "[]extractors.Extractor" + NETWORKRequestDoc.Fields[6].Description = "ReadAll determines if the data stream should be read till the end regardless of the size\n\nDefault value for read-all is false." + NETWORKRequestDoc.Fields[6].Comments[encoder.LineComment] = "ReadAll determines if the data stream should be read till the end regardless of the size" + + NETWORKRequestDoc.Fields[6].AddExample("", false) + NETWORKRequestDoc.Fields[7].Name = "matchers" + NETWORKRequestDoc.Fields[7].Type = "[]matchers.Matcher" NETWORKRequestDoc.Fields[7].Note = "" - NETWORKRequestDoc.Fields[7].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response." - NETWORKRequestDoc.Fields[7].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify" - NETWORKRequestDoc.Fields[8].Name = "matchers-condition" - NETWORKRequestDoc.Fields[8].Type = "string" + NETWORKRequestDoc.Fields[7].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument." + NETWORKRequestDoc.Fields[7].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify" + NETWORKRequestDoc.Fields[8].Name = "extractors" + NETWORKRequestDoc.Fields[8].Type = "[]extractors.Extractor" NETWORKRequestDoc.Fields[8].Note = "" - NETWORKRequestDoc.Fields[8].Description = "MatchersCondition is the condition between the matchers. Default is OR." - NETWORKRequestDoc.Fields[8].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR." - NETWORKRequestDoc.Fields[8].Values = []string{ + NETWORKRequestDoc.Fields[8].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response." + NETWORKRequestDoc.Fields[8].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify" + NETWORKRequestDoc.Fields[9].Name = "matchers-condition" + NETWORKRequestDoc.Fields[9].Type = "string" + NETWORKRequestDoc.Fields[9].Note = "" + NETWORKRequestDoc.Fields[9].Description = "MatchersCondition is the condition between the matchers. Default is OR." + NETWORKRequestDoc.Fields[9].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR." + NETWORKRequestDoc.Fields[9].Values = []string{ "and", "or", } From 4a59c3dd84d034c0711e4b480fbd7cc64657f1bb Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 25 Oct 2021 12:09:59 +0000 Subject: [PATCH 032/146] Auto Generate Syntax Docs + JSONSchema [Mon Oct 25 12:09:59 UTC 2021] :robot: --- nuclei-jsonschema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 57f8fc7f..1db207d7 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -850,6 +850,7 @@ ], "properties": { "id": { + "pattern": "^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$", "type": "string", "title": "id of the template", "description": "The Unique ID for the template", From ef3397b84de7a49569d2893700689de515a5715d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 17:42:01 +0530 Subject: [PATCH 033/146] Mark error for template syntax warning --- v2/pkg/parsers/parser.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 83059baa..7cd0c7b9 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -35,6 +35,7 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags [] } if validationError := validateTemplateFields(template); validationError != nil { + stats.Increment(SyntaxErrorStats) return false, validationError } From ca6cc513021d2e6a85aeb879055bc158c482fea8 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 17:47:39 +0530 Subject: [PATCH 034/146] Color change for warning disclaimer --- v2/internal/runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index b75f09c3..56bf4ea4 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -20,6 +20,6 @@ func showBanner() { gologger.Print().Msgf("%s\n", banner) gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") - gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n") - gologger.Error().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") + gologger.Print().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n") + gologger.Print().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") } From c6445519ecee46b0195a499d6746bb0a6bf372af Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 18:19:27 +0530 Subject: [PATCH 035/146] Fixed bug with github client and paths with no slash --- v2/pkg/reporting/trackers/github/github.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index 672f62ed..8d6cdb86 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -58,6 +58,9 @@ func New(options *Options) (*Integration, error) { if err != nil { return nil, errors.Wrap(err, "could not parse custom baseurl") } + if !strings.HasSuffix(parsed.Path, "/") { + parsed.Path += "/" + } client.BaseURL = parsed } return &Integration{client: client, options: options}, nil From 4a29443752b9f158cab15359dc09a3691262fca5 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 18:22:33 +0530 Subject: [PATCH 036/146] fix panic: Only generate curl command if request is not nil --- v2/pkg/protocols/http/request.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 1a619c17..3451e5f3 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -384,7 +384,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate }() var curlCommand string - if !request.Unsafe && resp != nil && generatedRequest.request != nil { + if !request.Unsafe && resp != nil && generatedRequest.request != nil && resp.Request != nil { bodyBytes, _ := generatedRequest.request.BodyBytes() resp.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes)) command, _ := http2curl.GetCurlCommand(resp.Request) From bfaea5b89d39f529b8dbf421e11fe2c61403df5c Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 18:35:45 +0530 Subject: [PATCH 037/146] Added more variables for DNS requests --- v2/go.mod | 1 + v2/go.sum | 2 ++ v2/pkg/protocols/dns/dns.go | 20 +++++++++++++++++++- v2/pkg/protocols/dns/dns_test.go | 11 +++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/v2/go.mod b/v2/go.mod index e5e74498..5e6e5f0c 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -53,6 +53,7 @@ require ( github.com/syndtr/goleveldb v1.0.0 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.1 + github.com/weppos/publicsuffix-go v0.15.0 github.com/xanzy/go-gitlab v0.50.3 github.com/ysmood/gson v0.6.4 // indirect github.com/ysmood/leakless v0.7.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index b95089c7..a497178f 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -750,6 +750,8 @@ github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/weppos/publicsuffix-go v0.15.0 h1:2uQCwDczZ8YZe5uD0mM3sXRoZYA74xxPuiKK8LdPcGQ= +github.com/weppos/publicsuffix-go v0.15.0/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xanzy/go-gitlab v0.50.3 h1:M7ncgNhCN4jaFNyXxarJhCLa9Qi6fdmCxFFhMTQPZiY= diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 15f42dc0..b5c27f7f 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -7,6 +7,8 @@ import ( "github.com/miekg/dns" "github.com/pkg/errors" + "github.com/weppos/publicsuffix-go/publicsuffix" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" @@ -132,7 +134,7 @@ func (request *Request) Make(domain string) (*dns.Msg, error) { var q dns.Question - final := replacer.Replace(request.Name, map[string]interface{}{"FQDN": domain}) + final := replacer.Replace(request.Name, generateDNSVariables(domain)) q.Name = dns.Fqdn(final) q.Qclass = request.class @@ -198,3 +200,19 @@ func classToInt(class string) uint16 { } return uint16(result) } + +func generateDNSVariables(domain string) map[string]interface{} { + parsed, err := publicsuffix.Parse(domain) + if err != nil { + return map[string]interface{}{"FQDN": domain} + } + + domainName := strings.Join([]string{parsed.SLD, parsed.TLD}, ".") + return map[string]interface{}{ + "{{FQDN}}": domain, + "{{RDN}}": domainName, + "{{DN}}": parsed.SLD, + "{{TLD}}": parsed.TLD, + "{{SD}}": parsed.TRD, + } +} diff --git a/v2/pkg/protocols/dns/dns_test.go b/v2/pkg/protocols/dns/dns_test.go index ae287984..74b38402 100644 --- a/v2/pkg/protocols/dns/dns_test.go +++ b/v2/pkg/protocols/dns/dns_test.go @@ -10,6 +10,17 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" ) +func TestGenerateDNSVariables(t *testing.T) { + vars := generateDNSVariables("www.projectdiscovery.io") + require.Equal(t, map[string]interface{}{ + "{{FQDN}}": "www.projectdiscovery.io", + "{{RDN}}": "projectdiscovery.io", + "{{DN}}": "projectdiscovery", + "{{TLD}}": "io", + "{{SD}}": "www", + }, vars, "could not get dns variables") +} + func TestDNSCompileMake(t *testing.T) { options := testutils.DefaultOptions From 09a2d88364c5cb5a36a242e28b5640d424ca055f Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 19:33:37 +0530 Subject: [PATCH 038/146] Misc --- v2/pkg/protocols/dns/dns.go | 10 +++++----- v2/pkg/protocols/dns/dns_test.go | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index b5c27f7f..66ed617a 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -209,10 +209,10 @@ func generateDNSVariables(domain string) map[string]interface{} { domainName := strings.Join([]string{parsed.SLD, parsed.TLD}, ".") return map[string]interface{}{ - "{{FQDN}}": domain, - "{{RDN}}": domainName, - "{{DN}}": parsed.SLD, - "{{TLD}}": parsed.TLD, - "{{SD}}": parsed.TRD, + "FQDN": domain, + "RDN": domainName, + "DN": parsed.SLD, + "TLD": parsed.TLD, + "SD": parsed.TRD, } } diff --git a/v2/pkg/protocols/dns/dns_test.go b/v2/pkg/protocols/dns/dns_test.go index 74b38402..85b00bd9 100644 --- a/v2/pkg/protocols/dns/dns_test.go +++ b/v2/pkg/protocols/dns/dns_test.go @@ -13,11 +13,11 @@ import ( func TestGenerateDNSVariables(t *testing.T) { vars := generateDNSVariables("www.projectdiscovery.io") require.Equal(t, map[string]interface{}{ - "{{FQDN}}": "www.projectdiscovery.io", - "{{RDN}}": "projectdiscovery.io", - "{{DN}}": "projectdiscovery", - "{{TLD}}": "io", - "{{SD}}": "www", + "FQDN": "www.projectdiscovery.io", + "RDN": "projectdiscovery.io", + "DN": "projectdiscovery", + "TLD": "io", + "SD": "www", }, vars, "could not get dns variables") } From ba108580c6fc539b372f224ff8a0b67a57a00db1 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 19:36:37 +0530 Subject: [PATCH 039/146] Fixed fqdn issue with tld parser --- v2/pkg/protocols/dns/dns.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 66ed617a..8aab88af 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -202,7 +202,7 @@ func classToInt(class string) uint16 { } func generateDNSVariables(domain string) map[string]interface{} { - parsed, err := publicsuffix.Parse(domain) + parsed, err := publicsuffix.Parse(strings.TrimSuffix(domain, ".")) if err != nil { return map[string]interface{}{"FQDN": domain} } From 6541b04f4c5a2d8e74ae1f0c5282033065cd9f9d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 23:24:42 +0530 Subject: [PATCH 040/146] Added new type and exclude-type flag --- v2/cmd/nuclei/main.go | 2 + v2/internal/runner/runner.go | 2 + v2/pkg/catalog/loader/filter/tag_filter.go | 39 ++++++++++++++- .../catalog/loader/filter/tag_filter_test.go | 49 +++++++++++++------ v2/pkg/catalog/loader/loader.go | 4 ++ v2/pkg/parsers/parser.go | 6 +-- v2/pkg/templates/templates.go | 18 +++++++ v2/pkg/types/types.go | 4 ++ 8 files changed, 104 insertions(+), 20 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index d2f8ebb9..99ef6558 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -68,6 +68,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), + flagSet.NormalizedStringSliceVarP(&options.Protocols, "type", "tp", []string{}, "protocol types to be executed (http,dns,headless,network,file,etc)"), + flagSet.NormalizedStringSliceVarP(&options.ExcludeProtocols, "exclude-type", "etype", []string{}, "protocol types to not be executed (http,dns,file,etc"), flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index b3f80695..c4c550ea 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -376,6 +376,8 @@ func (r *Runner) RunEnumeration() error { ExcludeSeverities: r.options.ExcludeSeverities, IncludeTags: r.options.IncludeTags, TemplatesDirectory: r.options.TemplatesDirectory, + Types: r.options.Protocols, + ExcludeTypes: r.options.ExcludeProtocols, Catalog: r.catalog, ExecutorOptions: executerOpts, } diff --git a/v2/pkg/catalog/loader/filter/tag_filter.go b/v2/pkg/catalog/loader/filter/tag_filter.go index ed8722c9..c33e96fd 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter.go +++ b/v2/pkg/catalog/loader/filter/tag_filter.go @@ -15,6 +15,8 @@ type TagFilter struct { authors map[string]struct{} block map[string]struct{} matchAllows map[string]struct{} + types map[string]struct{} + excludeTypes map[string]struct{} } // ErrExcluded is returned for excluded templates @@ -25,7 +27,7 @@ var ErrExcluded = errors.New("the template was excluded") // unless it is explicitly specified by user using the includeTags (matchAllows field). // Matching rule: (tag1 OR tag2...) AND (author1 OR author2...) AND (severity1 OR severity2...) AND (extraTags1 OR extraTags2...) // Returns true if the template matches the filter criteria, false otherwise. -func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string) (bool, error) { +func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string, templateType string) (bool, error) { for _, templateTag := range templateTags { _, blocked := tagFilter.block[templateTag] _, allowed := tagFilter.matchAllows[templateTag] @@ -51,6 +53,9 @@ func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templa return false, nil } + if !isTemplateTypeMatch(tagFilter, templateType) { + return false, nil + } return true, nil } @@ -116,6 +121,24 @@ func isTagMatch(tagFilter *TagFilter, templateTags []string) bool { return false } +func isTemplateTypeMatch(tagFilter *TagFilter, templateType string) bool { + if (len(tagFilter.excludeTypes) == 0 && len(tagFilter.types) == 0) || templateType == "" { + return true + } + + included := true + if len(tagFilter.types) > 0 { + _, included = tagFilter.types[templateType] + } + + excluded := false + if len(tagFilter.excludeTypes) > 0 { + _, excluded = tagFilter.excludeTypes[templateType] + } + + return included && !excluded +} + type Config struct { Tags []string ExcludeTags []string @@ -123,6 +146,8 @@ type Config struct { Severities severity.Severities ExcludeSeverities severity.Severities IncludeTags []string + Types []string + ExcludeTypes []string } // New returns a tag filter for nuclei tag based execution @@ -136,6 +161,8 @@ func New(config *Config) *TagFilter { excludeSeverities: make(map[severity.Severity]struct{}), block: make(map[string]struct{}), matchAllows: make(map[string]struct{}), + types: make(map[string]struct{}), + excludeTypes: make(map[string]struct{}), } for _, tag := range config.ExcludeTags { for _, val := range splitCommaTrim(tag) { @@ -177,6 +204,16 @@ func New(config *Config) *TagFilter { delete(filter.block, val) } } + for _, tag := range config.Types { + if _, ok := filter.types[tag]; !ok { + filter.types[tag] = struct{}{} + } + } + for _, tag := range config.ExcludeTypes { + if _, ok := filter.excludeTypes[tag]; !ok { + filter.excludeTypes[tag] = struct{}{} + } + } return filter } diff --git a/v2/pkg/catalog/loader/filter/tag_filter_test.go b/v2/pkg/catalog/loader/filter/tag_filter_test.go index 22d18b18..7599c27b 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter_test.go +++ b/v2/pkg/catalog/loader/filter/tag_filter_test.go @@ -15,19 +15,19 @@ func TestTagBasedFilter(t *testing.T) { }) t.Run("true", func(t *testing.T) { - matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil) + matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, "") require.True(t, matched, "could not get correct match") }) t.Run("false", func(t *testing.T) { - matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil) + matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil, "") require.False(t, matched, "could not get correct match") }) t.Run("match-extra-tags-positive", func(t *testing.T) { - matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}) + matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, "") require.True(t, matched, "could not get correct match") }) t.Run("match-extra-tags-negative", func(t *testing.T) { - matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}) + matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, "") require.False(t, matched, "could not get correct match") }) } @@ -36,7 +36,7 @@ func TestTagBasedFilter(t *testing.T) { filter := New(&Config{ ExcludeTags: []string{"dos"}, }) - matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil) + matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil, "") require.False(t, matched, "could not get correct match") require.Equal(t, ErrExcluded, err, "could not get correct error") }) @@ -46,7 +46,7 @@ func TestTagBasedFilter(t *testing.T) { ExcludeTags: []string{"dos", "fuzz"}, IncludeTags: []string{"fuzz"}, }) - matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil) + matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") require.Nil(t, err, "could not get match") require.True(t, matched, "could not get correct match") }) @@ -55,7 +55,7 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"fuzz"}, ExcludeTags: []string{"fuzz"}, }) - matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil) + matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") require.Nil(t, err, "could not get match") require.True(t, matched, "could not get correct match") }) @@ -63,24 +63,24 @@ func TestTagBasedFilter(t *testing.T) { filter := New(&Config{ Authors: []string{"pdteam"}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") require.True(t, matched, "could not get correct match") }) t.Run("match-severity", func(t *testing.T) { filter := New(&Config{ Severities: severity.Severities{severity.High}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "") require.True(t, matched, "could not get correct match") }) t.Run("match-exclude-severity", func(t *testing.T) { filter := New(&Config{ ExcludeSeverities: severity.Severities{severity.Low}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "") require.True(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil) + matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") require.False(t, matched, "could not get correct match") }) t.Run("match-exclude-with-tags", func(t *testing.T) { @@ -88,7 +88,7 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"tag"}, ExcludeTags: []string{"another"}, }) - matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil) + matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil, "") require.False(t, matched, "could not get correct match") }) t.Run("match-conditions", func(t *testing.T) { @@ -97,16 +97,33 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"jira"}, Severities: severity.Severities{severity.High}, }) - matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil) + matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil, "") require.True(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil) + matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, "") require.False(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil) + matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil, "") require.False(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil) + matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil, "") + require.False(t, matched, "could not get correct match") + }) + t.Run("match-type", func(t *testing.T) { + filter := New(&Config{ + Types: []string{"http"}, + }) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "http") + require.True(t, matched, "could not get correct match") + }) + t.Run("match-exclude-type", func(t *testing.T) { + filter := New(&Config{ + ExcludeTypes: []string{"http"}, + }) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "dns") + require.True(t, matched, "could not get correct match") + + matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "http") require.False(t, matched, "could not get correct match") }) } diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index eac22b4a..fe725e2e 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -21,6 +21,8 @@ type Config struct { Tags []string ExcludeTags []string + Types []string + ExcludeTypes []string Authors []string Severities severity.Severities ExcludeSeverities severity.Severities @@ -56,6 +58,8 @@ func New(config *Config) (*Store, error) { Severities: config.Severities, ExcludeSeverities: config.ExcludeSeverities, IncludeTags: config.IncludeTags, + Types: config.Types, + ExcludeTypes: config.ExcludeTypes, }), pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{ IncludedTemplates: config.IncludeTemplates, diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 7cd0c7b9..9e201fb0 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -39,7 +39,7 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags [] return false, validationError } - return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags) + return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags, template.Type()) } // LoadWorkflow returns true if the workflow is valid and matches the filtering criteria. @@ -59,12 +59,12 @@ func LoadWorkflow(templatePath string) (bool, error) { return false, nil } -func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string) (bool, error) { +func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string, templateType string) (bool, error) { templateTags := templateInfo.Tags.ToSlice() templateAuthors := templateInfo.Authors.ToSlice() templateSeverity := templateInfo.SeverityHolder.Severity - match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags) + match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags, templateType) if err == filter.ErrExcluded { return false, filter.ErrExcluded diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 61cb49ff..1e84def6 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -73,3 +73,21 @@ type Template struct { Path string `yaml:"-" json:"-"` } + +// Type returns the type of the template +func (t *Template) Type() string { + switch { + case len(t.RequestsDNS) > 0: + return "dns" + case len(t.RequestsFile) > 0: + return "file" + case len(t.RequestsHTTP) > 0: + return "http" + case len(t.RequestsHeadless) > 0: + return "headless" + case len(t.RequestsNetwork) > 0: + return "network" + default: + return "" + } +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index d258f424..36f66503 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -29,6 +29,10 @@ type Options struct { Severities severity.Severities // ExcludeSeverities specifies severities to exclude ExcludeSeverities severity.Severities + // Protocols contains the protocols to be allowed executed + Protocols goflags.NormalizedStringSlice + // ExcludeProtocols contains protocols to not be executed + ExcludeProtocols goflags.NormalizedStringSlice // Author filters templates based on their author and only run the matching ones. Author goflags.NormalizedStringSlice // IncludeTags includes specified tags to be run even while being in denylist From f0580298413c73e1647fb5c3e45610e95f799242 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 26 Oct 2021 16:29:00 +0530 Subject: [PATCH 041/146] Misc --- v2/cmd/integration-test/http.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index b2ecc816..c8bc6acc 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -41,8 +41,6 @@ type httpInteractshRequest struct{} func (h *httpInteractshRequest) Execute(filePath string) error { router := httprouter.New() router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) - value := r.Header.Get("url") if value != "" { if resp, _ := http.DefaultClient.Get(value); resp != nil { @@ -53,7 +51,7 @@ func (h *httpInteractshRequest) Execute(filePath string) error { ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } From 1dce8af04532f4fb53081441db7e3a65256bc9eb Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Tue, 26 Oct 2021 15:34:33 +0200 Subject: [PATCH 042/146] Linting issues --- v2/cmd/integration-test/loader.go | 10 ++++++++-- v2/pkg/catalog/loader/remote_loader.go | 4 ---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go index 13aa3f05..8c44bc20 100644 --- a/v2/cmd/integration-test/loader.go +++ b/v2/cmd/integration-test/loader.go @@ -35,7 +35,10 @@ func (h *remoteTemplateList) Execute(templateList string) error { if err != nil { w.WriteHeader(500) } - w.Write(file) + _, err = w.Write(file) + if err != nil { + w.WriteHeader(500) + } })) ts := httptest.NewServer(router) defer ts.Close() @@ -68,7 +71,10 @@ func (h *remoteWorkflowList) Execute(workflowList string) error { if err != nil { w.WriteHeader(500) } - w.Write(file) + _, err = w.Write(file) + if err != nil { + w.WriteHeader(500) + } })) ts := httptest.NewServer(router) defer ts.Close() diff --git a/v2/pkg/catalog/loader/remote_loader.go b/v2/pkg/catalog/loader/remote_loader.go index ec47ed51..d8f71eda 100644 --- a/v2/pkg/catalog/loader/remote_loader.go +++ b/v2/pkg/catalog/loader/remote_loader.go @@ -70,10 +70,6 @@ func getRemoteContent(URL string, w chan<- RemoteContentError, contentType Conte return } - if err != nil { - - } - scanner := bufio.NewScanner(response.Body) var templateList []string for scanner.Scan() { From a0318ffc8fce43a7524006a7bd1db7d08934b517 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 26 Oct 2021 20:36:44 +0530 Subject: [PATCH 043/146] Started refactor of template compilation + protocol building --- v2/pkg/engine/engine.go | 9 +++ v2/pkg/engine/engine_test.go | 1 + v2/pkg/templates/compile.go | 145 +++++++++++++++++++++-------------- 3 files changed, 98 insertions(+), 57 deletions(-) create mode 100644 v2/pkg/engine/engine.go create mode 100644 v2/pkg/engine/engine_test.go diff --git a/v2/pkg/engine/engine.go b/v2/pkg/engine/engine.go new file mode 100644 index 00000000..2bdb4a96 --- /dev/null +++ b/v2/pkg/engine/engine.go @@ -0,0 +1,9 @@ +package engine + +// Engine is an engine for running Nuclei Templates/Workflows. +// +// The engine contains multiple thread pools which allow using different +// concurrency values per protocol executed. This was something which was +// missing from the previous versions of nuclei. +type Engine struct { +} diff --git a/v2/pkg/engine/engine_test.go b/v2/pkg/engine/engine_test.go new file mode 100644 index 00000000..00a22ef6 --- /dev/null +++ b/v2/pkg/engine/engine_test.go @@ -0,0 +1 @@ +package engine diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index bf4e79b1..31446c9f 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "os" + "reflect" "strings" "github.com/pkg/errors" @@ -69,11 +70,6 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute options.TemplateInfo = template.Info options.TemplatePath = filePath - // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows) == 0 { - return nil, fmt.Errorf("no requests defined for %s", template.ID) - } - // Compile the workflow request if len(template.Workflows) > 0 { compiled := &template.Workflow @@ -83,56 +79,10 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute template.CompiledWorkflow.Options = &options } - // Compile the requests found - requests := []protocols.Request{} - if len(template.RequestsDNS) > 0 && !options.Options.OfflineHTTP { - for _, req := range template.RequestsDNS { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) + if err := template.compileProtocolRequests(options); err != nil { + return nil, err } - if len(template.RequestsHTTP) > 0 { - if options.Options.OfflineHTTP { - operatorsList := []*operators.Operators{} - mainLoop: - for _, req := range template.RequestsHTTP { - for _, path := range req.Path { - if !(strings.EqualFold(path, "{{BaseURL}}") || strings.EqualFold(path, "{{BaseURL}}/")) { - break mainLoop - } - } - operatorsList = append(operatorsList, &req.Operators) - } - if len(operatorsList) > 0 { - options.Operators = operatorsList - template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options) - } - } else { - for _, req := range template.RequestsHTTP { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - } - if len(template.RequestsFile) > 0 && !options.Options.OfflineHTTP { - for _, req := range template.RequestsFile { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - if len(template.RequestsNetwork) > 0 && !options.Options.OfflineHTTP { - for _, req := range template.RequestsNetwork { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - if len(template.RequestsHeadless) > 0 && !options.Options.OfflineHTTP && options.Options.Headless { - for _, req := range template.RequestsHeadless { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } if template.Executer != nil { if err := template.Executer.Compile(); err != nil { return nil, errors.Wrap(err, "could not compile request") @@ -151,14 +101,95 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute } // parseSelfContainedRequests parses the self contained template requests. -func (t *Template) parseSelfContainedRequests() { - if !t.SelfContained { +func (template *Template) parseSelfContainedRequests() { + if !template.SelfContained { return } - for _, request := range t.RequestsHTTP { + for _, request := range template.RequestsHTTP { request.SelfContained = true } - for _, request := range t.RequestsNetwork { + for _, request := range template.RequestsNetwork { request.SelfContained = true } } + +// Requests returns the total request count for the template +func (template *Template) Requests() int { + return len(template.RequestsDNS) + + len(template.RequestsHTTP) + + len(template.RequestsFile) + + len(template.RequestsNetwork) + + len(template.RequestsHeadless) + + len(template.Workflows) +} + +// compileProtocolRequests compiles all the protocol requests for the template +func (template *Template) compileProtocolRequests(options protocols.ExecuterOptions) error { + templateRequests := template.Requests() + + if templateRequests == 0 { + return fmt.Errorf("no requests defined for %s", template.ID) + } + + if template.Options.Options.OfflineHTTP { + template.compileOfflineHTTPRequest(options) + return nil + } + + var requests []protocols.Request + switch { + case len(template.RequestsDNS) > 0: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...) + + case len(template.RequestsFile) > 0: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...) + + case len(template.RequestsNetwork) > 0: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...) + + case len(template.RequestsHTTP) > 0: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...) + + case len(template.RequestsHeadless) > 0 && options.Options.Headless: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...) + } + template.Executer = executer.NewExecuter(requests, &options) + return nil +} + +func (template *Template) convertRequestToProtocolsRequest(requests interface{}) []protocols.Request { + switch reflect.TypeOf(requests).Kind() { + case reflect.Slice: + s := reflect.ValueOf(requests) + + requestSlice := make([]protocols.Request, s.Len()) + for i := 0; i < s.Len(); i++ { + value := s.Index(i) + valueInterface := value.Interface() + requestSlice[i] = valueInterface.(protocols.Request) + } + return requestSlice + } + return nil +} + +// compileOfflineHTTPRequest iterates all requests if offline http mode is +// specified and collects all matchers for all the base request templates +// (those with URL {{BaseURL}} and it's slash variation.) +func (template *Template) compileOfflineHTTPRequest(options protocols.ExecuterOptions) { + operatorsList := []*operators.Operators{} + +mainLoop: + for _, req := range template.RequestsHTTP { + for _, path := range req.Path { + if !(strings.EqualFold(path, "{{BaseURL}}") || strings.EqualFold(path, "{{BaseURL}}/")) { + break mainLoop + } + } + operatorsList = append(operatorsList, &req.Operators) + } + if len(operatorsList) > 0 { + options.Operators = operatorsList + template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options) + } +} From 2a84b9eb4443b9a826ff4d692d254eed2e5ea52d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 26 Oct 2021 20:40:02 +0530 Subject: [PATCH 044/146] misc --- v2/pkg/templates/compile.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 31446c9f..8c48308c 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -139,19 +139,19 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti var requests []protocols.Request switch { case len(template.RequestsDNS) > 0: - requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...) + requests = template.convertRequestToProtocolsRequest(template.RequestsDNS) case len(template.RequestsFile) > 0: - requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...) + requests = template.convertRequestToProtocolsRequest(template.RequestsFile) case len(template.RequestsNetwork) > 0: - requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...) + requests = template.convertRequestToProtocolsRequest(template.RequestsNetwork) case len(template.RequestsHTTP) > 0: - requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...) + requests = template.convertRequestToProtocolsRequest(template.RequestsHTTP) case len(template.RequestsHeadless) > 0 && options.Options.Headless: - requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...) + requests = template.convertRequestToProtocolsRequest(template.RequestsHeadless) } template.Executer = executer.NewExecuter(requests, &options) return nil From 97645dde520a0191cc87012e496720903a79bc43 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 27 Oct 2021 15:53:04 +0530 Subject: [PATCH 045/146] Added new workpool package + Misc refactor --- v2/cmd/nuclei/main.go | 2 + v2/internal/runner/options.go | 7 -- v2/internal/runner/runner.go | 145 +++++------------------------- v2/pkg/engine/engine.go | 7 ++ v2/pkg/engine/inputs/hmap/hmap.go | 120 +++++++++++++++++++++++++ v2/pkg/engine/workpool.go | 37 ++++++++ v2/pkg/templates/compile.go | 5 +- v2/pkg/templates/templates.go | 30 +++++++ v2/pkg/types/types.go | 4 + 9 files changed, 228 insertions(+), 129 deletions(-) create mode 100644 v2/pkg/engine/inputs/hmap/hmap.go create mode 100644 v2/pkg/engine/workpool.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index d2f8ebb9..13602bfc 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -110,6 +110,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"), flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"), flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"), + flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"), + flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "hc", 10, "maximum number of headless templates to be executed in parallel"), ) createGroup(flagSet, "optimization", "Optimizations", diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 59791fe6..55782174 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -47,13 +47,6 @@ func ParseOptions(options *types.Options) { gologger.Fatal().Msgf("Program exiting: %s\n", err) } - // Auto adjust rate limits when using headless mode if the user - // hasn't specified any custom limits. - if options.Headless && options.BulkSize == 25 && options.TemplateThreads == 10 { - options.BulkSize = 2 - options.TemplateThreads = 2 - } - // Load the resolvers if user asked for them loadResolvers(options) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index b3f80695..d4d7f40c 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -16,14 +16,12 @@ import ( "go.uber.org/ratelimit" "gopkg.in/yaml.v2" - "github.com/projectdiscovery/filekv" - "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/nuclei/v2/internal/colorizer" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" + "github.com/projectdiscovery/nuclei/v2/pkg/engine/inputs/hmap" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/parsers" @@ -46,22 +44,20 @@ import ( // Runner is a client for running the enumeration process. type Runner struct { - hostMap *hybrid.HybridMap - hostMapStream *filekv.FileDB - output output.Writer - interactsh *interactsh.Client - inputCount int64 - templatesConfig *config.Config - options *types.Options - projectFile *projectfile.ProjectFile - catalog *catalog.Catalog - progress progress.Progress - colorizer aurora.Aurora - issuesClient *reporting.Client - addColor func(severity.Severity) string - browser *engine.Browser - ratelimiter ratelimit.Limiter - hostErrors *hosterrorscache.Cache + output output.Writer + interactsh *interactsh.Client + templatesConfig *config.Config + options *types.Options + projectFile *projectfile.ProjectFile + catalog *catalog.Catalog + progress progress.Progress + colorizer aurora.Aurora + issuesClient *reporting.Client + addColor func(severity.Severity) string + hmapInputProvider *hmap.Input + browser *engine.Browser + ratelimiter ratelimit.Limiter + hostErrors *hosterrorscache.Cache } // New creates a new client for running enumeration process. @@ -116,103 +112,13 @@ func New(options *types.Options) (*Runner, error) { if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates { os.Exit(0) } - hm, err := hybrid.New(hybrid.DefaultDiskOptions) + + // Initialize the input source + hmapInput, err := hmap.New(options) if err != nil { - return nil, errors.Wrap(err, "could not create temporary input file") - } - runner.hostMap = hm - - if options.Stream { - fkvOptions := filekv.DefaultOptions - if tmpFileName, err := fileutil.GetTempFileName(); err != nil { - return nil, errors.Wrap(err, "could not create temporary input file") - } else { - fkvOptions.Path = tmpFileName - } - fkv, err := filekv.Open(fkvOptions) - if err != nil { - return nil, errors.Wrap(err, "could not create temporary unsorted input file") - } - runner.hostMapStream = fkv - } - - runner.inputCount = 0 - dupeCount := 0 - - // Handle multiple targets - if len(options.Targets) != 0 { - for _, target := range options.Targets { - url := strings.TrimSpace(target) - if url == "" { - continue - } - - if _, ok := runner.hostMap.Get(url); ok { - dupeCount++ - continue - } - - runner.inputCount++ - // nolint:errcheck // ignoring error - runner.hostMap.Set(url, nil) - if options.Stream { - _ = runner.hostMapStream.Set([]byte(url), nil) - } - } - } - - // Handle stdin - if options.Stdin { - scanner := bufio.NewScanner(os.Stdin) - for scanner.Scan() { - url := strings.TrimSpace(scanner.Text()) - if url == "" { - continue - } - - if _, ok := runner.hostMap.Get(url); ok { - dupeCount++ - continue - } - - runner.inputCount++ - // nolint:errcheck // ignoring error - runner.hostMap.Set(url, nil) - if options.Stream { - _ = runner.hostMapStream.Set([]byte(url), nil) - } - } - } - - // Handle target file - if options.TargetsFilePath != "" { - input, inputErr := os.Open(options.TargetsFilePath) - if inputErr != nil { - return nil, errors.Wrap(inputErr, "could not open targets file") - } - scanner := bufio.NewScanner(input) - for scanner.Scan() { - url := strings.TrimSpace(scanner.Text()) - if url == "" { - continue - } - if _, ok := runner.hostMap.Get(url); ok { - dupeCount++ - continue - } - runner.inputCount++ - // nolint:errcheck // ignoring error - runner.hostMap.Set(url, nil) - if options.Stream { - _ = runner.hostMapStream.Set([]byte(url), nil) - } - } - input.Close() - } - - if dupeCount > 0 { - gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", dupeCount) + return nil, errors.Wrap(err, "could not create input provider") } + runner.hmapInputProvider = hmapInput // Create the output file if asked outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.Output, options.TraceLogFile) @@ -312,13 +218,10 @@ func (r *Runner) Close() { if r.output != nil { r.output.Close() } - r.hostMap.Close() if r.projectFile != nil { r.projectFile.Close() } - if r.options.Stream { - r.hostMapStream.Close() - } + r.hmapInputProvider.Close() protocolinit.Close() } @@ -456,7 +359,7 @@ func (r *Runner) RunEnumeration() error { if len(template.Workflows) > 0 { continue } - unclusteredRequests += int64(template.TotalRequests) * r.inputCount + unclusteredRequests += int64(template.TotalRequests) * r.hmapInputProvider.Count() } if r.options.VerboseVerbose { @@ -509,7 +412,7 @@ func (r *Runner) RunEnumeration() error { if len(t.Workflows) > 0 { continue } - totalRequests += int64(t.TotalRequests) * r.inputCount + totalRequests += int64(t.TotalRequests) * r.hmapInputProvider.Count() } if totalRequests < unclusteredRequests { gologger.Info().Msgf("Templates clustered: %d (Reduced %d HTTP Requests)", clusterCount, unclusteredRequests-totalRequests) @@ -531,7 +434,7 @@ func (r *Runner) RunEnumeration() error { wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads) // tracks global progress and captures stdout/stderr until p.Wait finishes - r.progress.Init(r.inputCount, templateCount, totalRequests) + r.progress.Init(r.hmapInputProvider.Count(), templateCount, totalRequests) for _, t := range finalTemplates { wgtemplates.Add() diff --git a/v2/pkg/engine/engine.go b/v2/pkg/engine/engine.go index 2bdb4a96..978e845c 100644 --- a/v2/pkg/engine/engine.go +++ b/v2/pkg/engine/engine.go @@ -7,3 +7,10 @@ package engine // missing from the previous versions of nuclei. type Engine struct { } + +// InputProvider is an input provider interface for the nuclei execution +// engine. +// +// An example InputProvider is provided in form of hmap input provider. +type InputProvider interface { +} diff --git a/v2/pkg/engine/inputs/hmap/hmap.go b/v2/pkg/engine/inputs/hmap/hmap.go new file mode 100644 index 00000000..2910d045 --- /dev/null +++ b/v2/pkg/engine/inputs/hmap/hmap.go @@ -0,0 +1,120 @@ +// Package hmap implements a hybrid hmap/filekv backed input provider +// for nuclei that can either stream or store results using different kv stores. +package hmap + +import ( + "bufio" + "io" + "os" + "strings" + + "github.com/pkg/errors" + "github.com/projectdiscovery/filekv" + "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/hmap/store/hybrid" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Input is a hmap/filekv backed nuclei Input provider +type Input struct { + inputCount int64 + dupeCount int64 + hostMap *hybrid.HybridMap + hostMapStream *filekv.FileDB +} + +// New creates a new hmap backed nuclei Input Provider +// and initializes it based on the passed options Model. +func New(options *types.Options) (*Input, error) { + hm, err := hybrid.New(hybrid.DefaultDiskOptions) + if err != nil { + return nil, errors.Wrap(err, "could not create temporary input file") + } + + input := &Input{hostMap: hm} + if options.Stream { + fkvOptions := filekv.DefaultOptions + if tmpFileName, err := fileutil.GetTempFileName(); err != nil { + return nil, errors.Wrap(err, "could not create temporary input file") + } else { + fkvOptions.Path = tmpFileName + } + fkv, err := filekv.Open(fkvOptions) + if err != nil { + return nil, errors.Wrap(err, "could not create temporary unsorted input file") + } + input.hostMapStream = fkv + } + if initErr := input.initializeInputSources(options); initErr != nil { + return nil, initErr + } + if input.dupeCount > 0 { + gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", input.dupeCount) + } + return input, nil +} + +// Close closes the input provider +func (i *Input) Close() { + i.hostMap.Close() + if i.hostMapStream != nil { + i.hostMapStream.Close() + } +} + +// initializeInputSources initializes the input sources for hmap input +func (i *Input) initializeInputSources(options *types.Options) error { + // Handle targets flags + for _, target := range options.Targets { + i.normalizeStoreInputValue(target) + } + + // Handle stdin + if options.Stdin { + i.scanInputFromReader(os.Stdin) + } + + // Handle target file + if options.TargetsFilePath != "" { + input, inputErr := os.Open(options.TargetsFilePath) + if inputErr != nil { + return errors.Wrap(inputErr, "could not open targets file") + } + i.scanInputFromReader(input) + input.Close() + } + return nil +} + +// scanInputFromReader scans a line of input from reader and passes it for storage +func (i *Input) scanInputFromReader(reader io.Reader) { + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + i.normalizeStoreInputValue(scanner.Text()) + } +} + +// normalizeStoreInputValue normalizes and stores passed input values +func (i *Input) normalizeStoreInputValue(value string) { + url := strings.TrimSpace(value) + if url == "" { + return + } + + if _, ok := i.hostMap.Get(url); ok { + i.dupeCount++ + return + } + + i.inputCount++ + _ = i.hostMap.Set(url, nil) + if i.hostMapStream != nil { + _ = i.hostMapStream.Set([]byte(url), nil) + } +} + +// Count returns the input count +func (i *Input) Count() int64 { + return i.inputCount +} diff --git a/v2/pkg/engine/workpool.go b/v2/pkg/engine/workpool.go new file mode 100644 index 00000000..04db4471 --- /dev/null +++ b/v2/pkg/engine/workpool.go @@ -0,0 +1,37 @@ +package engine + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/templates" +) + +// WorkPool implements an execution pool for executing different +// types of task with different concurreny requirements. +// +// It also allows Configuration of such requirements. This is used +// for per-module like separate headless concurrency etc. +type WorkPool struct { + config WorkPoolConfig +} + +// WorkPoolConfig is the configuration for workpool +type WorkPoolConfig struct { + // InputConcurrency is the concurrency for inputs values. + InputConcurrency int + // TypeConcurrency is the concurrency for the request type templates. + TypeConcurrency int + // HeadlessInputConcurrency is the concurrency for headless inputs values. + HeadlessInputConcurrency int + // TypeConcurrency is the concurrency for the headless request type templates. + HeadlessTypeConcurrency int +} + +// New returns a new WorkPool instance +func New(config WorkPoolConfig) *WorkPool { + return &WorkPool{config: config} +} + +// TODO: port these invocations of waitgroups and input logic into a generic +// workpool type functionality. +func (w *WorkPool) Execute(templates []*templates.Template) { + +} diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 8c48308c..22c2d764 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -131,7 +131,7 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti return fmt.Errorf("no requests defined for %s", template.ID) } - if template.Options.Options.OfflineHTTP { + if options.Options.OfflineHTTP { template.compileOfflineHTTPRequest(options) return nil } @@ -157,6 +157,9 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti return nil } +// convertRequestToProtocolsRequest is a convenience wrapper to convert +// arbitrary interfaces which are slices of requests from the template to a +// slice of protocols.Request interface items. func (template *Template) convertRequestToProtocolsRequest(requests interface{}) []protocols.Request { switch reflect.TypeOf(requests).Kind() { case reflect.Slice: diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 61cb49ff..7e0e42a3 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -73,3 +73,33 @@ type Template struct { Path string `yaml:"-" json:"-"` } + +// TemplateTypes is a list of accepted template types +var TemplateTypes = []string{ + "dns", + "file", + "http", + "headless", + "network", + "workflow", +} + +// Type returns the type of the template +func (t *Template) Type() string { + switch { + case len(t.RequestsDNS) > 0: + return "dns" + case len(t.RequestsFile) > 0: + return "file" + case len(t.RequestsHTTP) > 0: + return "http" + case len(t.RequestsHeadless) > 0: + return "headless" + case len(t.RequestsNetwork) > 0: + return "network" + case len(t.Workflows) > 0: + return "workflow" + default: + return "" + } +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index d258f424..9f8f784b 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -77,6 +77,10 @@ type Options struct { BulkSize int // TemplateThreads is the number of templates executed in parallel TemplateThreads int + // HeadlessBulkSize is the of targets analyzed in parallel for each headless template + HeadlessBulkSize int + // HeadlessTemplateThreads is the number of headless templates executed in parallel + HeadlessTemplateThreads int // Timeout is the seconds to wait for a response from the server. Timeout int // Retries is the number of times to retry the request From c16c93fe7ca698d3086b85fd86e0c1e8c99a222e Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 27 Oct 2021 16:50:36 +0530 Subject: [PATCH 046/146] refactor the modules to core --- v2/internal/runner/runner.go | 46 +++------------- v2/pkg/core/engine.go | 53 +++++++++++++++++++ v2/pkg/core/engine_test.go | 1 + .../processor.go => pkg/core/execute.go} | 46 +++++++++++++++- v2/pkg/{engine => core}/inputs/hmap/hmap.go | 0 .../execute.go => core/workflow_execute.go} | 12 ++--- .../workflow_execute_test.go} | 39 +++++++------- v2/pkg/{engine => core}/workpool.go | 6 +-- v2/pkg/engine/engine.go | 16 ------ v2/pkg/engine/engine_test.go | 1 - v2/pkg/protocols/protocols.go | 6 +++ v2/pkg/templates/workflows.go | 14 +---- 12 files changed, 139 insertions(+), 101 deletions(-) create mode 100644 v2/pkg/core/engine.go create mode 100644 v2/pkg/core/engine_test.go rename v2/{internal/runner/processor.go => pkg/core/execute.go} (61%) rename v2/pkg/{engine => core}/inputs/hmap/hmap.go (100%) rename v2/pkg/{workflows/execute.go => core/workflow_execute.go} (95%) rename v2/pkg/{workflows/execute_test.go => core/workflow_execute_test.go} (77%) rename v2/pkg/{engine => core}/workpool.go (90%) delete mode 100644 v2/pkg/engine/engine.go delete mode 100644 v2/pkg/engine/engine_test.go diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index d4d7f40c..100e364f 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -2,7 +2,6 @@ package runner import ( "bufio" - "fmt" "os" "path/filepath" "strings" @@ -11,7 +10,6 @@ import ( "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/remeh/sizedwaitgroup" - "github.com/rs/xid" "go.uber.org/atomic" "go.uber.org/ratelimit" "gopkg.in/yaml.v2" @@ -21,6 +19,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" + "github.com/projectdiscovery/nuclei/v2/pkg/core" "github.com/projectdiscovery/nuclei/v2/pkg/engine/inputs/hmap" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -28,7 +27,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" @@ -247,6 +245,9 @@ func (r *Runner) RunEnumeration() error { cache = hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount).SetVerbose(r.options.Verbose) } r.hostErrors = cache + + // Create the executer options which will be used throughout the execution + // stage by the nuclei engine modules. executerOpts := protocols.ExecuterOptions{ Output: r.output, Options: r.options, @@ -259,12 +260,13 @@ func (r *Runner) RunEnumeration() error { Browser: r.browser, HostErrorsCache: cache, } + engine := core.New(r.options) + engine.SetExecuterOptions(executerOpts) workflowLoader, err := parsers.NewLoader(&executerOpts) if err != nil { return errors.Wrap(err, "Could not create loader.") } - executerOpts.WorkflowLoader = workflowLoader loaderConfig := loader.Config{ @@ -370,42 +372,6 @@ func (r *Runner) RunEnumeration() error { r.logAvailableTemplate(template.Path) } } - templatesMap := make(map[string]*templates.Template) - for _, v := range store.Templates() { - templatesMap[v.Path] = v - } - originalTemplatesCount := len(store.Templates()) - clusterCount := 0 - clusters := clusterer.Cluster(templatesMap) - for _, cluster := range clusters { - if len(cluster) > 1 && !r.options.OfflineHTTP { - executerOpts := protocols.ExecuterOptions{ - Output: r.output, - Options: r.options, - Progress: r.progress, - Catalog: r.catalog, - RateLimiter: r.ratelimiter, - IssuesClient: r.issuesClient, - Browser: r.browser, - ProjectFile: r.projectFile, - Interactsh: r.interactsh, - HostErrorsCache: cache, - } - clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) - - finalTemplates = append(finalTemplates, &templates.Template{ - ID: clusterID, - RequestsHTTP: cluster[0].RequestsHTTP, - Executer: clusterer.NewExecuter(cluster, &executerOpts), - TotalRequests: len(cluster[0].RequestsHTTP), - }) - clusterCount += len(cluster) - } else { - finalTemplates = append(finalTemplates, cluster...) - } - } - - finalTemplates = append(finalTemplates, store.Workflows()...) var totalRequests int64 for _, t := range finalTemplates { diff --git a/v2/pkg/core/engine.go b/v2/pkg/core/engine.go new file mode 100644 index 00000000..d7fdc6bb --- /dev/null +++ b/v2/pkg/core/engine.go @@ -0,0 +1,53 @@ +package core + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Engine is an engine for running Nuclei Templates/Workflows. +// +// The engine contains multiple thread pools which allow using different +// concurrency values per protocol executed. +// +// The engine does most of the heavy lifting of execution, from clustering +// templates to leading to the final execution by the workpool, it is +// handled by the engine. +type Engine struct { + workPool *WorkPool + options *types.Options + executerOpts protocols.ExecuterOptions +} + +// InputProvider is an input provider interface for the nuclei execution +// engine. +// +// An example InputProvider is provided in form of hmap input provider. +type InputProvider interface { +} + +// New returns a new Engine instance +func New(options *types.Options) *Engine { + workPool := NewWorkPool(WorkPoolConfig{ + InputConcurrency: options.BulkSize, + TypeConcurrency: options.TemplateThreads, + HeadlessInputConcurrency: options.HeadlessBulkSize, + HeadlessTypeConcurrency: options.HeadlessTemplateThreads, + }) + engine := &Engine{ + options: options, + workPool: workPool, + } + return engine +} + +// SetExecuterOptions sets the executer options for the engine. This is required +// before using the engine to perform any execution. +func (e *Engine) SetExecuterOptions(options protocols.ExecuterOptions) { + e.executerOpts = options +} + +// ExecuterOptions returns protocols.ExecuterOptions for nuclei engine. +func (e *Engine) ExecuterOptions() protocols.ExecuterOptions { + return e.executerOpts +} diff --git a/v2/pkg/core/engine_test.go b/v2/pkg/core/engine_test.go new file mode 100644 index 00000000..9a8bc959 --- /dev/null +++ b/v2/pkg/core/engine_test.go @@ -0,0 +1 @@ +package core diff --git a/v2/internal/runner/processor.go b/v2/pkg/core/execute.go similarity index 61% rename from v2/internal/runner/processor.go rename to v2/pkg/core/execute.go index 6b14b5f9..65c31558 100644 --- a/v2/internal/runner/processor.go +++ b/v2/pkg/core/execute.go @@ -1,5 +1,48 @@ -package runner +package core +import ( + "fmt" + + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/rs/xid" +) + +// clusterTemplates performs identical http requests clustering for a list of templates +func (e *Engine) clusterTemplates(templatesList []*templates.Template) ([]*templates.Template, int) { + if e.options.OfflineHTTP { + return templatesList, 0 + } + + templatesMap := make(map[string]*templates.Template) + for _, v := range templatesList { + templatesMap[v.Path] = v + } + clusterCount := 0 + + finalTemplatesList := make([]*templates.Template, 0, len(templatesList)) + clusters := clusterer.Cluster(templatesMap) + for _, cluster := range clusters { + if len(cluster) > 1 { + executerOpts := e.ExecuterOptions() + + clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) + + finalTemplatesList = append(finalTemplatesList, &templates.Template{ + ID: clusterID, + RequestsHTTP: cluster[0].RequestsHTTP, + Executer: clusterer.NewExecuter(cluster, &executerOpts), + TotalRequests: len(cluster[0].RequestsHTTP), + }) + clusterCount += len(cluster) + } else { + finalTemplatesList = append(finalTemplatesList, cluster...) + } + } + return finalTemplatesList, clusterCount +} + +/* import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/templates" @@ -79,3 +122,4 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool { wg.Wait() return results.Load() } +*/ diff --git a/v2/pkg/engine/inputs/hmap/hmap.go b/v2/pkg/core/inputs/hmap/hmap.go similarity index 100% rename from v2/pkg/engine/inputs/hmap/hmap.go rename to v2/pkg/core/inputs/hmap/hmap.go diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/core/workflow_execute.go similarity index 95% rename from v2/pkg/workflows/execute.go rename to v2/pkg/core/workflow_execute.go index c0710c6a..59a5ade5 100644 --- a/v2/pkg/workflows/execute.go +++ b/v2/pkg/core/workflow_execute.go @@ -1,12 +1,6 @@ -package workflows - -import ( - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/atomic" -) +package core +/* // RunWorkflow runs a workflow on an input and returns true or false func (w *Workflow) RunWorkflow(input string) bool { results := &atomic.Bool{} @@ -123,4 +117,4 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res } } return mainErr -} +}*/ diff --git a/v2/pkg/workflows/execute_test.go b/v2/pkg/core/workflow_execute_test.go similarity index 77% rename from v2/pkg/workflows/execute_test.go rename to v2/pkg/core/workflow_execute_test.go index 6d9ab6a0..62cd4084 100644 --- a/v2/pkg/workflows/execute_test.go +++ b/v2/pkg/core/workflow_execute_test.go @@ -1,5 +1,6 @@ -package workflows +package core +/* import ( "testing" @@ -10,13 +11,14 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) func TestWorkflowsSimple(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, }} @@ -29,13 +31,13 @@ func TestWorkflowsSimpleMultiple(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, - {Executers: []*ProtocolExecuterPair{{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, @@ -53,14 +55,14 @@ func TestWorkflowsSubtemplates(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ {OperatorsResult: &operators.Result{}, Results: []*output.ResultEvent{{}}}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, - }, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + }, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, @@ -78,12 +80,12 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: false, executeHook: func(input string) { firstInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, - }, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + }, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, @@ -101,8 +103,8 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ @@ -111,7 +113,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { Extracts: map[string][]string{}, }}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, - }, Matchers: []*Matcher{{Name: "tomcat", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + }, Matchers: []*workflows.Matcher{{Name: "tomcat", Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, @@ -129,8 +131,8 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ @@ -139,7 +141,7 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { Extracts: map[string][]string{}, }}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, - }, Matchers: []*Matcher{{Name: "apache", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + }, Matchers: []*workflows.Matcher{{Name: "apache", Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, @@ -187,3 +189,4 @@ func (m *mockExecuter) ExecuteWithResults(input string, callback protocols.Outpu } return nil } +*/ diff --git a/v2/pkg/engine/workpool.go b/v2/pkg/core/workpool.go similarity index 90% rename from v2/pkg/engine/workpool.go rename to v2/pkg/core/workpool.go index 04db4471..f307b205 100644 --- a/v2/pkg/engine/workpool.go +++ b/v2/pkg/core/workpool.go @@ -1,4 +1,4 @@ -package engine +package core import ( "github.com/projectdiscovery/nuclei/v2/pkg/templates" @@ -25,8 +25,8 @@ type WorkPoolConfig struct { HeadlessTypeConcurrency int } -// New returns a new WorkPool instance -func New(config WorkPoolConfig) *WorkPool { +// NewWorkPool returns a new WorkPool instance +func NewWorkPool(config WorkPoolConfig) *WorkPool { return &WorkPool{config: config} } diff --git a/v2/pkg/engine/engine.go b/v2/pkg/engine/engine.go deleted file mode 100644 index 978e845c..00000000 --- a/v2/pkg/engine/engine.go +++ /dev/null @@ -1,16 +0,0 @@ -package engine - -// Engine is an engine for running Nuclei Templates/Workflows. -// -// The engine contains multiple thread pools which allow using different -// concurrency values per protocol executed. This was something which was -// missing from the previous versions of nuclei. -type Engine struct { -} - -// InputProvider is an input provider interface for the nuclei execution -// engine. -// -// An example InputProvider is provided in form of hmap input provider. -type InputProvider interface { -} diff --git a/v2/pkg/engine/engine_test.go b/v2/pkg/engine/engine_test.go deleted file mode 100644 index 00a22ef6..00000000 --- a/v2/pkg/engine/engine_test.go +++ /dev/null @@ -1 +0,0 @@ -package engine diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index d77ba147..83a76f7c 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -64,6 +64,12 @@ type ExecuterOptions struct { WorkflowLoader model.WorkflowLoader } +// Copy returns a copy of the executeroptions structure +func (e ExecuterOptions) Copy() ExecuterOptions { + copy := e + return copy +} + // Request is an interface implemented any protocol based request generator. type Request interface { // Compile compiles the request generators preparing any requests possible. diff --git a/v2/pkg/templates/workflows.go b/v2/pkg/templates/workflows.go index 9d510f2e..246c4bf8 100644 --- a/v2/pkg/templates/workflows.go +++ b/v2/pkg/templates/workflows.go @@ -62,19 +62,7 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr return nil } for _, path := range paths { - opts := protocols.ExecuterOptions{ - Output: options.Output, - Options: options.Options, - Progress: options.Progress, - Catalog: options.Catalog, - Browser: options.Browser, - RateLimiter: options.RateLimiter, - IssuesClient: options.IssuesClient, - Interactsh: options.Interactsh, - ProjectFile: options.ProjectFile, - HostErrorsCache: options.HostErrorsCache, - } - template, err := Parse(path, preprocessor, opts) + template, err := Parse(path, preprocessor, options.Copy()) if err != nil { gologger.Warning().Msgf("Could not parse workflow template %s: %v\n", path, err) continue From df78ea72c5c266fecc4c01e97a700025a921ed42 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 27 Oct 2021 18:41:39 +0530 Subject: [PATCH 047/146] misc --- v2/pkg/core/execute.go | 24 ++++++++++++++++++++++++ v2/pkg/core/workpool.go | 2 -- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index 65c31558..112be4e2 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -8,6 +8,30 @@ import ( "github.com/rs/xid" ) +// Execute takes a list of templates/workflows that have been compiled +// and executes them based on provided concurrency options. +// +// All the execution logic for the templates/workflows happens in this part +// of the engine. +func (e *Engine) Execute(templates []*templates.Template) { + finalTemplates, clusterCount := e.clusterTemplates(templates) + + for _, template := range finalTemplates { + templateType := template.Type() + + switch { + case template.SelfContained: + // Self Contained requests are executed here + + case templateType == "workflow": + // Workflows requests are executed here + + default: + // All other request types are executed here + } + } +} + // clusterTemplates performs identical http requests clustering for a list of templates func (e *Engine) clusterTemplates(templatesList []*templates.Template) ([]*templates.Template, int) { if e.options.OfflineHTTP { diff --git a/v2/pkg/core/workpool.go b/v2/pkg/core/workpool.go index f307b205..ef50f70f 100644 --- a/v2/pkg/core/workpool.go +++ b/v2/pkg/core/workpool.go @@ -30,8 +30,6 @@ func NewWorkPool(config WorkPoolConfig) *WorkPool { return &WorkPool{config: config} } -// TODO: port these invocations of waitgroups and input logic into a generic -// workpool type functionality. func (w *WorkPool) Execute(templates []*templates.Template) { } From a539184ffd9701368f93be55197a22a99639d79e Mon Sep 17 00:00:00 2001 From: kchason Date: Wed, 27 Oct 2021 12:11:42 -0400 Subject: [PATCH 048/146] Switch logic to a shared package --- .../protocols/headless/engine/http_client.go | 26 ++------------ .../http/httpclientpool/clientpool.go | 26 ++------------ v2/pkg/protocols/utils/utils.go | 34 +++++++++++++++++++ 3 files changed, 40 insertions(+), 46 deletions(-) create mode 100644 v2/pkg/protocols/utils/utils.go diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index d2f48c59..0edf6bf6 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -2,9 +2,7 @@ package engine import ( "crypto/tls" - "crypto/x509" - "io/ioutil" - "log" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" "net/http" "net/url" "time" @@ -23,26 +21,8 @@ func newhttpClient(options *types.Options) *http.Client { InsecureSkipVerify: true, } - // Build the TLS config with the client certificate if it has been configured with the appropriate options. - // Only one of the options needs to be checked since the validation checks in main.go ensure that all three - // files are set if any of the client certification configuration options are. - if len(options.ClientCertFile) > 0 { - // Load the client certificate using the PEM encoded client certificate and the private key file - cert, err := tls.LoadX509KeyPair(options.ClientCertFile, options.ClientKeyFile) - if err != nil { - log.Fatal(err) - } - tlsConfig.Certificates = []tls.Certificate{cert} - - // Load the certificate authority PEM certificate into the TLS configuration - caCert, err := ioutil.ReadFile(options.ClientCAFile) - if err != nil { - log.Fatal(err) - } - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - tlsConfig.RootCAs = caCertPool - } + // Add the client certificate authentication to the request if it's configured + tlsConfig = utils.AddConfiguredClientCertToRequest(tlsConfig, options) transport := &http.Transport{ DialContext: dialer.Dial, diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 3b0ce9b4..2b7dc024 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -3,10 +3,8 @@ package httpclientpool import ( "context" "crypto/tls" - "crypto/x509" "fmt" - "io/ioutil" - "log" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" "net" "net/http" "net/http/cookiejar" @@ -170,26 +168,8 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl InsecureSkipVerify: true, } - // Build the TLS config with the client certificate if it has been configured with the appropriate options. - // Only one of the options needs to be checked since the validation checks in main.go ensure that all three - // files are set if any of the client certification configuration options are. - if len(options.ClientCertFile) > 0 { - // Load the client certificate using the PEM encoded client certificate and the private key file - cert, err := tls.LoadX509KeyPair(options.ClientCertFile, options.ClientKeyFile) - if err != nil { - log.Fatal(err) - } - tlsConfig.Certificates = []tls.Certificate{cert} - - // Load the certificate authority PEM certificate into the TLS configuration - caCert, err := ioutil.ReadFile(options.ClientCAFile) - if err != nil { - log.Fatal(err) - } - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - tlsConfig.RootCAs = caCertPool - } + // Add the client certificate authentication to the request if it's configured + tlsConfig = utils.AddConfiguredClientCertToRequest(tlsConfig, options) transport := &http.Transport{ DialContext: Dialer.Dial, diff --git a/v2/pkg/protocols/utils/utils.go b/v2/pkg/protocols/utils/utils.go new file mode 100644 index 00000000..8d43fd47 --- /dev/null +++ b/v2/pkg/protocols/utils/utils.go @@ -0,0 +1,34 @@ +package utils + +import ( + "crypto/tls" + "crypto/x509" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "io/ioutil" + "log" +) + +// AddConfiguredClientCertToRequest adds the client certificate authentication to the tls.Config object and returns it +func AddConfiguredClientCertToRequest(tlsConfig *tls.Config, options *types.Options) *tls.Config { + // Build the TLS config with the client certificate if it has been configured with the appropriate options. + // Only one of the options needs to be checked since the validation checks in main.go ensure that all three + // files are set if any of the client certification configuration options are. + if len(options.ClientCertFile) > 0 { + // Load the client certificate using the PEM encoded client certificate and the private key file + cert, err := tls.LoadX509KeyPair(options.ClientCertFile, options.ClientKeyFile) + if err != nil { + log.Fatal(err) + } + tlsConfig.Certificates = []tls.Certificate{cert} + + // Load the certificate authority PEM certificate into the TLS configuration + caCert, err := ioutil.ReadFile(options.ClientCAFile) + if err != nil { + log.Fatal(err) + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + tlsConfig.RootCAs = caCertPool + } + return tlsConfig +} From d124dbacc7cf3b3ce4328c391a908cedd68703d6 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 28 Oct 2021 17:20:07 +0530 Subject: [PATCH 049/146] Moved all important execution stuff to engine --- v2/internal/runner/runner.go | 43 ++---- v2/pkg/core/engine.go | 4 + v2/pkg/core/execute.go | 162 +++++++++----------- v2/pkg/core/inputs/{hmap => hybrid}/hmap.go | 17 +- v2/pkg/core/workflow_execute.go | 29 ++-- v2/pkg/core/workflow_execute_test.go | 30 ++-- v2/pkg/core/workpool.go | 38 ++++- v2/pkg/protocols/protocols.go | 2 + 8 files changed, 171 insertions(+), 154 deletions(-) rename v2/pkg/core/inputs/{hmap => hybrid}/hmap.go (88%) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 100e364f..84f2837a 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -9,8 +9,6 @@ import ( "github.com/logrusorgru/aurora" "github.com/pkg/errors" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/atomic" "go.uber.org/ratelimit" "gopkg.in/yaml.v2" @@ -20,7 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v2/pkg/core" - "github.com/projectdiscovery/nuclei/v2/pkg/engine/inputs/hmap" + "github.com/projectdiscovery/nuclei/v2/pkg/engine/inputs/hybrid" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/parsers" @@ -34,7 +32,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/reporting" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" @@ -52,7 +49,7 @@ type Runner struct { colorizer aurora.Aurora issuesClient *reporting.Client addColor func(severity.Severity) string - hmapInputProvider *hmap.Input + hmapInputProvider *hybrid.Input browser *engine.Browser ratelimiter ratelimit.Limiter hostErrors *hosterrorscache.Cache @@ -112,7 +109,7 @@ func New(options *types.Options) (*Runner, error) { } // Initialize the input source - hmapInput, err := hmap.New(options) + hmapInput, err := hybrid.New(options) if err != nil { return nil, errors.Wrap(err, "could not create input provider") } @@ -259,6 +256,7 @@ func (r *Runner) RunEnumeration() error { ProjectFile: r.projectFile, Browser: r.browser, HostErrorsCache: cache, + Colorizer: r.colorizer, } engine := core.New(r.options) engine.SetExecuterOptions(executerOpts) @@ -351,9 +349,6 @@ func (r *Runner) RunEnumeration() error { gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows())) } - // pre-parse all the templates, apply filters - finalTemplates := []*templates.Template{} - var unclusteredRequests int64 for _, template := range store.Templates() { // workflows will dynamically adjust the totals while running, as @@ -373,6 +368,12 @@ func (r *Runner) RunEnumeration() error { } } + // Cluster the templates first because we want info on how many + // templates did we cluster for showing to user in CLI + originalTemplatesCount := len(store.Templates()) + finalTemplates, clusterCount := engine.ClusterTemplates(store.Templates()) + finalTemplates = append(finalTemplates, store.Workflows()...) + var totalRequests int64 for _, t := range finalTemplates { if len(t.Workflows) > 0 { @@ -391,32 +392,10 @@ func (r *Runner) RunEnumeration() error { return errors.New("no valid templates were found") } - /* - TODO does it make sense to run the logic below if there are no targets specified? - Can we safely assume the user is just experimenting with the template/workflow filters before running them? - */ - - results := &atomic.Bool{} - wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads) - // tracks global progress and captures stdout/stderr until p.Wait finishes r.progress.Init(r.hmapInputProvider.Count(), templateCount, totalRequests) - for _, t := range finalTemplates { - wgtemplates.Add() - go func(template *templates.Template) { - defer wgtemplates.Done() - - if template.SelfContained { - results.CAS(false, r.processSelfContainedTemplates(template)) - } else if len(template.Workflows) > 0 { - results.CAS(false, r.processWorkflowWithList(template)) - } else { - results.CAS(false, r.processTemplateWithList(template)) - } - }(t) - } - wgtemplates.Wait() + results := engine.ExecuteWithOpts(finalTemplates, r.hmapInputProvider, true) if r.interactsh != nil { matched := r.interactsh.Close() diff --git a/v2/pkg/core/engine.go b/v2/pkg/core/engine.go index d7fdc6bb..45554624 100644 --- a/v2/pkg/core/engine.go +++ b/v2/pkg/core/engine.go @@ -24,6 +24,10 @@ type Engine struct { // // An example InputProvider is provided in form of hmap input provider. type InputProvider interface { + // Count returns the number of items for input provider + Count() int64 + // Scan calls a callback function till the input provider is exhausted + Scan(callback func(value string)) } // New returns a new Engine instance diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index 112be4e2..3593123a 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -3,9 +3,12 @@ package core import ( "fmt" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/remeh/sizedwaitgroup" "github.com/rs/xid" + "go.uber.org/atomic" ) // Execute takes a list of templates/workflows that have been compiled @@ -13,27 +16,86 @@ import ( // // All the execution logic for the templates/workflows happens in this part // of the engine. -func (e *Engine) Execute(templates []*templates.Template) { - finalTemplates, clusterCount := e.clusterTemplates(templates) +func (e *Engine) Execute(templates []*templates.Template, input InputProvider) *atomic.Bool { + return e.ExecuteWithOpts(templates, input, false) +} +// ExecuteWithOpts is execute with the full options +func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, input InputProvider, noCluster bool) *atomic.Bool { + var finalTemplates []*templates.Template + if !noCluster { + finalTemplates, _ = e.ClusterTemplates(templatesList) + } else { + finalTemplates = templatesList + } + + results := &atomic.Bool{} for _, template := range finalTemplates { templateType := template.Type() + var wg *sizedwaitgroup.SizedWaitGroup + if templateType == "headless" { + wg = e.workPool.Headless + } else { + wg = e.workPool.Default + } + + wg.Add() switch { case template.SelfContained: - // Self Contained requests are executed here - - case templateType == "workflow": - // Workflows requests are executed here - + // Self Contained requests are executed here separately + e.executeSelfContainedTemplateWithInput(template, results) default: // All other request types are executed here + e.executeModelWithInput(templateType, template, input, results) } } + e.workPool.Wait() + return results } -// clusterTemplates performs identical http requests clustering for a list of templates -func (e *Engine) clusterTemplates(templatesList []*templates.Template) ([]*templates.Template, int) { +// processSelfContainedTemplates execute a self-contained template. +func (e *Engine) executeSelfContainedTemplateWithInput(template *templates.Template, results *atomic.Bool) { + match, err := template.Executer.Execute("") + if err != nil { + gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err) + } + results.CAS(false, match) +} + +// executeModelWithInput executes a type of template with input +func (e *Engine) executeModelWithInput(templateType string, template *templates.Template, input InputProvider, results *atomic.Bool) { + wg := e.workPool.InputPool(templateType) + + input.Scan(func(scannedValue string) { + // Skip if the host has had errors + if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue) { + return + } + + wg.Waitgroup.Add() + go func(value string) { + defer wg.Waitgroup.Done() + + var match bool + var err error + switch templateType { + case "workflow": + match = e.executeWorkflow(value, template.CompiledWorkflow) + default: + match, err = template.Executer.Execute(value) + } + if err != nil { + gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err) + } + results.CAS(false, match) + }(scannedValue) + }) + wg.Waitgroup.Wait() +} + +// ClusterTemplates performs identical http requests clustering for a list of templates +func (e *Engine) ClusterTemplates(templatesList []*templates.Template) ([]*templates.Template, int) { if e.options.OfflineHTTP { return templatesList, 0 } @@ -65,85 +127,3 @@ func (e *Engine) clusterTemplates(templatesList []*templates.Template) ([]*templ } return finalTemplatesList, clusterCount } - -/* -import ( - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/atomic" -) - -// processSelfContainedTemplates execute a self-contained template. -func (r *Runner) processSelfContainedTemplates(template *templates.Template) bool { - match, err := template.Executer.Execute("") - if err != nil { - gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) - } - return match -} - -// processTemplateWithList execute a template against the list of user provided targets -func (r *Runner) processTemplateWithList(template *templates.Template) bool { - results := &atomic.Bool{} - wg := sizedwaitgroup.New(r.options.BulkSize) - processItem := func(k, _ []byte) error { - URL := string(k) - - // Skip if the host has had errors - if r.hostErrors != nil && r.hostErrors.Check(URL) { - return nil - } - wg.Add() - go func(URL string) { - defer wg.Done() - - match, err := template.Executer.Execute(URL) - if err != nil { - gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) - } - results.CAS(false, match) - }(URL) - return nil - } - if r.options.Stream { - _ = r.hostMapStream.Scan(processItem) - } else { - r.hostMap.Scan(processItem) - } - - wg.Wait() - return results.Load() -} - -// processTemplateWithList process a template on the URL list -func (r *Runner) processWorkflowWithList(template *templates.Template) bool { - results := &atomic.Bool{} - wg := sizedwaitgroup.New(r.options.BulkSize) - - processItem := func(k, _ []byte) error { - URL := string(k) - - // Skip if the host has had errors - if r.hostErrors != nil && r.hostErrors.Check(URL) { - return nil - } - wg.Add() - go func(URL string) { - defer wg.Done() - match := template.CompiledWorkflow.RunWorkflow(URL) - results.CAS(false, match) - }(URL) - return nil - } - - if r.options.Stream { - _ = r.hostMapStream.Scan(processItem) - } else { - r.hostMap.Scan(processItem) - } - - wg.Wait() - return results.Load() -} -*/ diff --git a/v2/pkg/core/inputs/hmap/hmap.go b/v2/pkg/core/inputs/hybrid/hmap.go similarity index 88% rename from v2/pkg/core/inputs/hmap/hmap.go rename to v2/pkg/core/inputs/hybrid/hmap.go index 2910d045..1eb4f5e4 100644 --- a/v2/pkg/core/inputs/hmap/hmap.go +++ b/v2/pkg/core/inputs/hybrid/hmap.go @@ -1,6 +1,6 @@ -// Package hmap implements a hybrid hmap/filekv backed input provider +// Package hybrid implements a hybrid hmap/filekv backed input provider // for nuclei that can either stream or store results using different kv stores. -package hmap +package hybrid import ( "bufio" @@ -118,3 +118,16 @@ func (i *Input) normalizeStoreInputValue(value string) { func (i *Input) Count() int64 { return i.inputCount } + +// Scan calls an input provider till the callback is exhausted +func (i *Input) Scan(callback func(value string)) { + callbackFunc := func(k, _ []byte) error { + callback(string(k)) + return nil + } + if i.hostMapStream != nil { + _ = i.hostMapStream.Scan(callbackFunc) + } else { + i.hostMap.Scan(callbackFunc) + } +} diff --git a/v2/pkg/core/workflow_execute.go b/v2/pkg/core/workflow_execute.go index 59a5ade5..8c255d7a 100644 --- a/v2/pkg/core/workflow_execute.go +++ b/v2/pkg/core/workflow_execute.go @@ -1,15 +1,22 @@ package core -/* -// RunWorkflow runs a workflow on an input and returns true or false -func (w *Workflow) RunWorkflow(input string) bool { +import ( + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/workflows" + "github.com/remeh/sizedwaitgroup" + "go.uber.org/atomic" +) + +// executeWorkflow runs a workflow on an input and returns true or false +func (e *Engine) executeWorkflow(input string, w *workflows.Workflow) bool { results := &atomic.Bool{} swg := sizedwaitgroup.New(w.Options.Options.TemplateThreads) for _, template := range w.Workflows { swg.Add() - func(template *WorkflowTemplate) { - if err := w.runWorkflowStep(template, input, results, &swg); err != nil { + func(template *workflows.WorkflowTemplate) { + if err := e.runWorkflowStep(template, input, results, &swg, w); err != nil { gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) } swg.Done() @@ -21,7 +28,7 @@ func (w *Workflow) RunWorkflow(input string) bool { // runWorkflowStep runs a workflow step for the workflow. It executes the workflow // in a recursive manner running all subtemplates and matchers. -func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup) error { +func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input string, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error { var firstMatched bool var err error var mainErr error @@ -84,8 +91,8 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res for _, subtemplate := range matcher.Subtemplates { swg.Add() - go func(subtemplate *WorkflowTemplate) { - if err := w.runWorkflowStep(subtemplate, input, results, swg); err != nil { + go func(subtemplate *workflows.WorkflowTemplate) { + if err := e.runWorkflowStep(subtemplate, input, results, swg, w); err != nil { gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", subtemplate.Template, err) } swg.Done() @@ -108,8 +115,8 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res for _, subtemplate := range template.Subtemplates { swg.Add() - go func(template *WorkflowTemplate) { - if err := w.runWorkflowStep(template, input, results, swg); err != nil { + go func(template *workflows.WorkflowTemplate) { + if err := e.runWorkflowStep(template, input, results, swg, w); err != nil { gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) } swg.Done() @@ -117,4 +124,4 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res } } return mainErr -}*/ +} diff --git a/v2/pkg/core/workflow_execute_test.go b/v2/pkg/core/workflow_execute_test.go index 62cd4084..a00ce604 100644 --- a/v2/pkg/core/workflow_execute_test.go +++ b/v2/pkg/core/workflow_execute_test.go @@ -1,6 +1,5 @@ package core -/* import ( "testing" @@ -17,13 +16,14 @@ import ( func TestWorkflowsSimple(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) - workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.True(t, matched, "could not get correct match value") } @@ -31,7 +31,7 @@ func TestWorkflowsSimpleMultiple(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input @@ -44,7 +44,8 @@ func TestWorkflowsSimpleMultiple(t *testing.T) { }}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.True(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -55,7 +56,7 @@ func TestWorkflowsSubtemplates(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input @@ -69,7 +70,8 @@ func TestWorkflowsSubtemplates(t *testing.T) { }}}}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.True(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -80,7 +82,7 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: false, executeHook: func(input string) { firstInput = input @@ -92,7 +94,8 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { }}}}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.False(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -103,7 +106,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input @@ -120,7 +123,8 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { }}}}}}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.True(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -148,7 +152,8 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { }}}}}}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.False(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -189,4 +194,3 @@ func (m *mockExecuter) ExecuteWithResults(input string, callback protocols.Outpu } return nil } -*/ diff --git a/v2/pkg/core/workpool.go b/v2/pkg/core/workpool.go index ef50f70f..23b50d68 100644 --- a/v2/pkg/core/workpool.go +++ b/v2/pkg/core/workpool.go @@ -1,7 +1,7 @@ package core import ( - "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/remeh/sizedwaitgroup" ) // WorkPool implements an execution pool for executing different @@ -10,7 +10,9 @@ import ( // It also allows Configuration of such requirements. This is used // for per-module like separate headless concurrency etc. type WorkPool struct { - config WorkPoolConfig + Headless *sizedwaitgroup.SizedWaitGroup + Default *sizedwaitgroup.SizedWaitGroup + config WorkPoolConfig } // WorkPoolConfig is the configuration for workpool @@ -27,9 +29,35 @@ type WorkPoolConfig struct { // NewWorkPool returns a new WorkPool instance func NewWorkPool(config WorkPoolConfig) *WorkPool { - return &WorkPool{config: config} + headlessWg := sizedwaitgroup.New(config.HeadlessTypeConcurrency) + defaultWg := sizedwaitgroup.New(config.TypeConcurrency) + + return &WorkPool{ + config: config, + Headless: &headlessWg, + Default: &defaultWg, + } } -func (w *WorkPool) Execute(templates []*templates.Template) { - +// Wait waits for all the workpool waitgroups to finish +func (w *WorkPool) Wait() { + w.Default.Wait() + w.Headless.Wait() +} + +// InputWorkPool is a workpool per-input +type InputWorkPool struct { + Waitgroup *sizedwaitgroup.SizedWaitGroup +} + +// InputPool returns a workpool for an input type +func (w *WorkPool) InputPool(templateType string) *InputWorkPool { + var count int + if templateType == "headless" { + count = w.config.HeadlessInputConcurrency + } else { + count = w.config.InputConcurrency + } + swg := sizedwaitgroup.New(count) + return &InputWorkPool{Waitgroup: &swg} } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 83a76f7c..d877328f 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -3,6 +3,7 @@ package protocols import ( "go.uber.org/ratelimit" + "github.com/logrusorgru/aurora" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/operators" @@ -61,6 +62,7 @@ type ExecuterOptions struct { Operators []*operators.Operators // only used by offlinehttp module + Colorizer aurora.Aurora WorkflowLoader model.WorkflowLoader } From 0abc7202b1c4cd12da364100a2309ef8a9ca2ae1 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 28 Oct 2021 17:45:38 +0530 Subject: [PATCH 050/146] Misc fixes with goflags --- v2/go.mod | 73 +++++++++++++++++------------- v2/go.sum | 86 +++++++++++++++++++++--------------- v2/internal/runner/runner.go | 2 +- v2/pkg/core/execute.go | 1 + 4 files changed, 96 insertions(+), 66 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index e5e74498..bbce9b15 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -5,93 +5,99 @@ go 1.17 require ( github.com/Ice3man543/nvd v1.0.8 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible - github.com/akrylysov/pogreb v0.10.1 // indirect - github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c + github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 github.com/andygrunwald/go-jira v1.14.0 - github.com/antchfx/htmlquery v1.2.3 + github.com/antchfx/htmlquery v1.2.4 github.com/apex/log v1.9.0 github.com/blang/semver v3.5.1+incompatible github.com/bluele/gcache v0.0.2 - github.com/c4milo/unpackit v0.1.0 // indirect github.com/corpix/uarand v0.1.1 - github.com/go-rod/rod v0.101.7 + github.com/go-rod/rod v0.101.8 github.com/google/go-github v17.0.0+incompatible - github.com/gosuri/uilive v0.0.4 // indirect - github.com/gosuri/uiprogress v0.0.1 // indirect - github.com/itchyny/gojq v0.12.4 + github.com/itchyny/gojq v0.12.5 github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.16.1 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/mattn/go-runewidth v0.0.13 // indirect github.com/miekg/dns v1.1.43 github.com/olekukonko/tablewriter v0.0.5 github.com/owenrumney/go-sarif v1.0.11 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 - github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e + github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 - github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 + github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a github.com/projectdiscovery/gologger v1.1.4 - github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa + github.com/projectdiscovery/hmap v0.0.2-0.20210825180603-fca7166c158f github.com/projectdiscovery/interactsh v0.0.6 - github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 + github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df github.com/projectdiscovery/rawhttp v0.0.7 - github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a + github.com/projectdiscovery/retryabledns v1.0.12 github.com/projectdiscovery/retryablehttp-go v1.0.2 - github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d + github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/yamldoc-go v1.0.2 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/segmentio/ksuid v1.0.4 - github.com/shirou/gopsutil/v3 v3.21.7 + github.com/shirou/gopsutil/v3 v3.21.9 github.com/spaolacci/murmur3 v1.1.0 github.com/spf13/cast v1.4.1 github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.1 - github.com/xanzy/go-gitlab v0.50.3 - github.com/ysmood/gson v0.6.4 // indirect - github.com/ysmood/leakless v0.7.0 // indirect + github.com/xanzy/go-gitlab v0.51.1 go.uber.org/atomic v1.9.0 go.uber.org/multierr v1.7.0 go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 - golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab - golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect + golang.org/x/net v0.0.0-20211020060615-d418f374d309 + golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 golang.org/x/text v0.3.7 - google.golang.org/appengine v1.6.7 // indirect gopkg.in/yaml.v2 v2.4.0 moul.io/http2curl v1.0.0 ) require ( git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect + github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect + github.com/DataDog/zstd v1.4.8 // indirect github.com/PuerkitoBio/goquery v1.6.0 // indirect github.com/StackExchange/wmi v1.2.1 // indirect + github.com/akrylysov/pogreb v0.10.1 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect - github.com/antchfx/xpath v1.1.6 // indirect - github.com/aymerick/douceur v0.2.0 // indirect + github.com/antchfx/xpath v1.2.0 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect + github.com/c4milo/unpackit v0.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect + github.com/cockroachdb/errors v1.8.6 // indirect + github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect + github.com/cockroachdb/pebble v0.0.0-20210827150156-ff43a5880feb // indirect + github.com/cockroachdb/redact v1.1.3 // indirect + github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/badger v1.6.2 // indirect + github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dsnet/compress v0.0.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect github.com/eggsampler/acme/v3 v3.2.1 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/go-ole/go-ole v1.2.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect + github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/gorilla/css v1.0.0 // indirect + github.com/gosuri/uilive v0.0.4 // indirect + github.com/gosuri/uiprogress v0.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect @@ -100,8 +106,10 @@ require ( github.com/karlseguin/ccache/v2 v2.0.8 // indirect github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/pgzip v1.2.5 // indirect + github.com/kr/pretty v0.3.0 // indirect + github.com/kr/text v0.2.0 // indirect github.com/mattn/go-isatty v0.0.13 // indirect - github.com/microcosm-cc/bluemonday v1.0.15 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -111,17 +119,22 @@ require ( github.com/projectdiscovery/mapcidr v0.0.8 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/rivo/uniseg v0.2.0 // indirect - github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect - github.com/tklauser/go-sysconf v0.3.7 // indirect - github.com/tklauser/numcpus v0.2.3 // indirect + github.com/rogpeppe/go-internal v1.8.0 // indirect + github.com/tklauser/go-sysconf v0.3.9 // indirect + github.com/tklauser/numcpus v0.3.0 // indirect github.com/trivago/tgo v1.0.7 // indirect github.com/ulikunitz/xz v0.5.10 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect github.com/ysmood/goob v0.3.0 // indirect + github.com/ysmood/gson v0.6.4 // indirect + github.com/ysmood/leakless v0.7.0 // indirect github.com/zclconf/go-cty v1.8.4 // indirect go.etcd.io/bbolt v1.3.6 // indirect + golang.org/x/exp v0.0.0-20210826195003-46c773283d9d // indirect + golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect + google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect diff --git a/v2/go.sum b/v2/go.sum index b95089c7..6b184d9f 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -35,12 +35,14 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7 git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Ice3man543/nvd v1.0.8 h1:2CBEgOxyWAkQocnnmEMmRtVPWooPRvcuHFLWj48EM4c= github.com/Ice3man543/nvd v1.0.8/go.mod h1:0DxLJk6revOcJKiZxa2K+rNF/HO1zJO97lqQtXhXfSc= @@ -67,8 +69,9 @@ github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= -github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c h1:oJsq4z4xKgZWWOhrSZuLZ5KyYfRFytddLL1E5+psfIY= github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= +github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 h1:NjwIgLQlD46o79bheVG4SCdRnnOz4XtgUN1WABX5DLA= +github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -79,10 +82,12 @@ github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5z github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andygrunwald/go-jira v1.14.0 h1:7GT/3qhar2dGJ0kq8w0d63liNyHOnxZsUZ9Pe4+AKBI= github.com/andygrunwald/go-jira v1.14.0/go.mod h1:KMo2f4DgMZA1C9FdImuLc04x4WQhn5derQpnsuBFgqE= -github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M= github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= -github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0= +github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494= +github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc= github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8= +github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= @@ -96,14 +101,11 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= -github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= -github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -124,8 +126,10 @@ github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -140,13 +144,18 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= +github.com/cockroachdb/errors v1.8.6 h1:Am9evxl/po3RzpokemQvq7S7Cd0mxv24xy0B/trlQF4= github.com/cockroachdb/errors v1.8.6/go.mod h1:hOm5fabihW+xEyY1kuypGwqT+Vt7rafg04ytBtIpeIQ= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/cockroachdb/pebble v0.0.0-20210728210723-48179f1d4dae/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= +github.com/cockroachdb/pebble v0.0.0-20210827150156-ff43a5880feb h1:ZkYzWJexCQZyR1ejLy1IHvgdXTMe3Hiyvwlkyk5xr5Y= github.com/cockroachdb/pebble v0.0.0-20210827150156-ff43a5880feb/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/redact v1.1.1/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= @@ -172,12 +181,15 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= @@ -185,6 +197,7 @@ github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -216,6 +229,7 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -232,8 +246,8 @@ github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI= -github.com/go-rod/rod v0.101.7 h1:kbI5CNvcRhf7feybBln4xDutsM0mbsF0ENNZfKcF6WA= -github.com/go-rod/rod v0.101.7/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY= +github.com/go-rod/rod v0.101.8 h1:oV0O97uwjkCVyAP0hD6K6bBE8FUMIjs0dtF7l6kEBsU= +github.com/go-rod/rod v0.101.8/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -247,11 +261,13 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -328,8 +344,6 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= -github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -385,8 +399,9 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/ github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA= -github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o= github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg= +github.com/itchyny/gojq v0.12.5 h1:6SJ1BQ1VAwJAlIvLSIZmqHP/RUEq3qfVWvsRxrqhsD0= +github.com/itchyny/gojq v0.12.5/go.mod h1:3e1hZXv+Kwvdp6V9HXpVrvddiHVApi5EDZwS+zLFeiE= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= @@ -479,8 +494,6 @@ github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go. github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= -github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY= -github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -555,6 +568,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -572,9 +586,8 @@ github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydK github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= +github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542 h1:wvHtITg+Y2eMVdN/4OR8wx7747RFTZX/UcxxJOE35F4= github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542/go.mod h1:TuapmLiqtunJOxpM7g0tpTy/TUF/0S+XFyx0B0Wx0DQ= -github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges= -github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ= github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew= github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ= github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= @@ -584,16 +597,16 @@ github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 h1:b7zDUSsgN5f4/IlhKF6RVGsp/NkHIuty0o1YjzAMKUs= github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= +github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a h1:EzwVm8i4zmzqZX55vrDtyfogwHh8AAZ3cWCJe4fEduk= +github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= github.com/projectdiscovery/hmap v0.0.2-0.20210727180307-d63d35146e97/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= +github.com/projectdiscovery/hmap v0.0.2-0.20210825180603-fca7166c158f h1:lLBOzyKoIZTUoMIi3hxa8fj6EtK0EozlftfR4A8Mc4w= github.com/projectdiscovery/hmap v0.0.2-0.20210825180603-fca7166c158f/go.mod h1:RLM8b1z2HEq74u5AXN1Lbvfq+1BZWpnTQJcwLnMLA54= -github.com/projectdiscovery/hmap v0.0.2-0.20210917073634-bfb0e9c03800/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= -github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0= -github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= github.com/projectdiscovery/interactsh v0.0.6 h1:z2nVuc2FDRtOVjoYySiG0by/JA1Dwmgzb2rBFJEFR/c= github.com/projectdiscovery/interactsh v0.0.6/go.mod h1:dB/c1A9I2trIHfMbU/wNzxQkGara0UE33zDtGcdBh+U= @@ -609,23 +622,21 @@ github.com/projectdiscovery/mapcidr v0.0.8 h1:16U05F2x3o/jSTsxSCY2hCuCs9xOSwVxjo github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= -github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 h1:SNtAiRRrJtDJJDroaa/bFXt/Tix2LA6+rHRib0ORlJQ= -github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= +github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df h1:CvTNAUD5JbLMqpMFoGNgfk2gOcN0NC57ICu0+oK84vs= +github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY= github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg= github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= +github.com/projectdiscovery/retryabledns v1.0.12 h1:OzCsUaipN75OwjtH62FxBIhKye1NmnfG4DxtQclOtns= github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= -github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a h1:WJQjr9qi/VjWhdNiGyNqcFi0967Gp0W3I769bCpHOJE= -github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U= github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d h1:YBYwsm8MrSp9t7mLehyqGwUKZWB08fG+YRePQRo5iFw= -github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d/go.mod h1:JK4F9ACNPgO+Lbm80khX2q1ABInBMbwIOmbsEE61Sn4= github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -664,8 +675,6 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= -github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -674,8 +683,9 @@ github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.21.7 h1:PnTqQamUjwEDSgn+nBGu0qSDV/CfvyiR/gwTH3i7HTU= github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4= +github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bsYrHg= +github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -726,10 +736,12 @@ github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPf github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible h1:guTq1YxwB8XSILkI9q4IrOmrCOS6Hc1L3hmOhi4Swcs= github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible/go.mod h1:waFwwyiAhGey2e+dNoYQ/iLhIcFqhCW7zL/+vDU1WLo= -github.com/tklauser/go-sysconf v0.3.7 h1:HT7h4+536gjqeq1ZIJPgOl1rg1XFatQGVZWp7Py53eg= github.com/tklauser/go-sysconf v0.3.7/go.mod h1:JZIdXh4RmBvZDBZ41ld2bGxRV3n4daiiqA3skYhAoQ4= -github.com/tklauser/numcpus v0.2.3 h1:nQ0QYpiritP6ViFhrKYsiv6VVxOpum2Gks5GhnJbS/8= +github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/numcpus v0.2.3/go.mod h1:vpEPS/JC+oZGGQ/My/vJnNsvMDQL6PwOqt8dsCw5j+E= +github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM= github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= @@ -752,8 +764,9 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= -github.com/xanzy/go-gitlab v0.50.3 h1:M7ncgNhCN4jaFNyXxarJhCLa9Qi6fdmCxFFhMTQPZiY= github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= +github.com/xanzy/go-gitlab v0.51.1 h1:wWKLalwx4omxFoHh3PLs9zDgAD4GXDP/uoxwMRCSiWM= +github.com/xanzy/go-gitlab v0.51.1/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -846,6 +859,7 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/exp v0.0.0-20210826195003-46c773283d9d h1:0jjV4zcNl3aKvtqwdWq0orBLGpQB9TZ9qH45oQhI5MM= golang.org/x/exp v0.0.0-20210826195003-46c773283d9d/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -922,16 +936,17 @@ golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI= +golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab h1:llrcWN/wOwO+6gAyfBzxb5hZ+c3mriU/0+KNgYu6adA= golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 h1:B333XXssMuKQeBwiNODx4TupZy7bf4sxFZnN2ZOcvUE= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1006,9 +1021,10 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw= -golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e h1:XMgFehsDnnLGtjvjOfqWSUzt0alpTR1RSEuznObga2c= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 84f2837a..4f8ef975 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -18,7 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v2/pkg/core" - "github.com/projectdiscovery/nuclei/v2/pkg/engine/inputs/hybrid" + "github.com/projectdiscovery/nuclei/v2/pkg/core/inputs/hybrid" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/parsers" diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index 3593123a..88370787 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -49,6 +49,7 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, input Inpu // All other request types are executed here e.executeModelWithInput(templateType, template, input, results) } + wg.Done() } e.workPool.Wait() return results From 533fb3f1080a395a7799969dfc5ff71ca0cdaa93 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 28 Oct 2021 22:02:22 +0530 Subject: [PATCH 051/146] misc flag update --- v2/cmd/nuclei/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 99ef6558..85f0ecf3 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -68,8 +68,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), - flagSet.NormalizedStringSliceVarP(&options.Protocols, "type", "tp", []string{}, "protocol types to be executed (http,dns,headless,network,file,etc)"), - flagSet.NormalizedStringSliceVarP(&options.ExcludeProtocols, "exclude-type", "etype", []string{}, "protocol types to not be executed (http,dns,file,etc"), + flagSet.NormalizedStringSliceVarP(&options.Protocols, "type", "pt", []string{}, "protocol types to be executed (http,dns,headless,network,file,etc)"), + flagSet.NormalizedStringSliceVarP(&options.ExcludeProtocols, "exclude-type", "ept", []string{}, "protocol types to not be executed (http,dns,file,etc"), flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), ) From 1ca2cf3beace4bb5b876b50e8b13a4289c7091f9 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 28 Oct 2021 23:17:05 +0530 Subject: [PATCH 052/146] Misc --- v2/internal/runner/runner.go | 118 +++++++++++++++----------------- v2/pkg/catalog/loader/loader.go | 21 ++++++ 2 files changed, 75 insertions(+), 64 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 4f8ef975..d27713a3 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -267,22 +267,7 @@ func (r *Runner) RunEnumeration() error { } executerOpts.WorkflowLoader = workflowLoader - loaderConfig := loader.Config{ - Templates: r.options.Templates, - Workflows: r.options.Workflows, - ExcludeTemplates: r.options.ExcludedTemplates, - Tags: r.options.Tags, - ExcludeTags: r.options.ExcludeTags, - IncludeTemplates: r.options.IncludeTemplates, - Authors: r.options.Author, - Severities: r.options.Severities, - ExcludeSeverities: r.options.ExcludeSeverities, - IncludeTags: r.options.IncludeTags, - TemplatesDirectory: r.options.TemplatesDirectory, - Catalog: r.catalog, - ExecutorOptions: executerOpts, - } - store, err := loader.New(&loaderConfig) + store, err := loader.New(loader.NewConfig(r.options, r.catalog, executerOpts)) if err != nil { return errors.Wrap(err, "could not load templates from config") } @@ -300,54 +285,7 @@ func (r *Runner) RunEnumeration() error { return nil // exit } - // Display stats for any loaded templates' syntax warnings or errors - stats.Display(parsers.SyntaxWarningStats) - stats.Display(parsers.SyntaxErrorStats) - - builder := &strings.Builder{} - if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" { - builder.WriteString(" (") - - if strings.Contains(config.Version, "-dev") { - builder.WriteString(r.colorizer.Blue("development").String()) - } else if config.Version == r.templatesConfig.NucleiLatestVersion { - builder.WriteString(r.colorizer.Green("latest").String()) - } else { - builder.WriteString(r.colorizer.Red("outdated").String()) - } - builder.WriteString(")") - } - messageStr := builder.String() - builder.Reset() - - gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr) - - if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { // TODO extract duplicated logic - builder.WriteString(" (") - - if r.templatesConfig.TemplateVersion == r.templatesConfig.NucleiTemplatesLatestVersion { - builder.WriteString(r.colorizer.Green("latest").String()) - } else { - builder.WriteString(r.colorizer.Red("outdated").String()) - } - builder.WriteString(")") - } - messageStr = builder.String() - builder.Reset() - - if r.templatesConfig != nil { - gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.TemplateVersion, messageStr) - } - if r.interactsh != nil { - gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL) - } - if len(store.Templates()) > 0 { - gologger.Info().Msgf("Templates added in last update: %d", r.countNewTemplates()) - gologger.Info().Msgf("Templates loaded for scan: %d", len(store.Templates())) - } - if len(store.Workflows()) > 0 { - gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows())) - } + r.displayExecutionInfo(store) var unclusteredRequests int64 for _, template := range store.Templates() { @@ -417,6 +355,58 @@ func (r *Runner) RunEnumeration() error { return nil } +// displayExecutionInfo displays misc info about the nuclei engine execution +func (r *Runner) displayExecutionInfo(store *loader.Store) { + // Display stats for any loaded templates' syntax warnings or errors + stats.Display(parsers.SyntaxWarningStats) + stats.Display(parsers.SyntaxErrorStats) + + builder := &strings.Builder{} + if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" { + builder.WriteString(" (") + + if strings.Contains(config.Version, "-dev") { + builder.WriteString(r.colorizer.Blue("development").String()) + } else if config.Version == r.templatesConfig.NucleiLatestVersion { + builder.WriteString(r.colorizer.Green("latest").String()) + } else { + builder.WriteString(r.colorizer.Red("outdated").String()) + } + builder.WriteString(")") + } + messageStr := builder.String() + builder.Reset() + + gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr) + + if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { // TODO extract duplicated logic + builder.WriteString(" (") + + if r.templatesConfig.TemplateVersion == r.templatesConfig.NucleiTemplatesLatestVersion { + builder.WriteString(r.colorizer.Green("latest").String()) + } else { + builder.WriteString(r.colorizer.Red("outdated").String()) + } + builder.WriteString(")") + } + messageStr = builder.String() + builder.Reset() + + if r.templatesConfig != nil { + gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.TemplateVersion, messageStr) + } + if r.interactsh != nil { + gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL) + } + if len(store.Templates()) > 0 { + gologger.Info().Msgf("Templates added in last update: %d", r.countNewTemplates()) + gologger.Info().Msgf("Templates loaded for scan: %d", len(store.Templates())) + } + if len(store.Workflows()) > 0 { + gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows())) + } +} + // readNewTemplatesFile reads newly added templates from directory if it exists func (r *Runner) readNewTemplatesFile() ([]string, error) { if r.templatesConfig == nil { diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index eac22b4a..386898b1 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/parsers" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Config contains the configuration options for the loader @@ -44,6 +45,26 @@ type Store struct { preprocessor templates.Preprocessor } +// NewConfig returns a new loader config +func NewConfig(options *types.Options, catalog *catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config { + loaderConfig := Config{ + Templates: options.Templates, + Workflows: options.Workflows, + ExcludeTemplates: options.ExcludedTemplates, + Tags: options.Tags, + ExcludeTags: options.ExcludeTags, + IncludeTemplates: options.IncludeTemplates, + Authors: options.Author, + Severities: options.Severities, + ExcludeSeverities: options.ExcludeSeverities, + IncludeTags: options.IncludeTags, + TemplatesDirectory: options.TemplatesDirectory, + Catalog: catalog, + ExecutorOptions: executerOpts, + } + return &loaderConfig +} + // New creates a new template store based on provided configuration func New(config *Config) (*Store, error) { // Create a tag filter based on provided configuration From 5393cc4cd509c95eda6dc7e543f5384feec09268 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 29 Oct 2021 03:19:43 +0530 Subject: [PATCH 053/146] Adjusting packages for more API-type design --- v2/internal/testutils/testutils.go | 38 +------------------ v2/pkg/core/inputs/inputs.go | 17 +++++++++ v2/pkg/output/output.go | 35 +++++++++++++++++ v2/pkg/progress/progress.go | 24 ++++++++++++ .../protocols/common/interactsh/interactsh.go | 14 +++++++ v2/pkg/types/types.go | 14 +++++++ 6 files changed, 105 insertions(+), 37 deletions(-) create mode 100644 v2/pkg/core/inputs/inputs.go diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index d62b1d63..e58c7079 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -1,7 +1,6 @@ package testutils import ( - "github.com/logrusorgru/aurora" "go.uber.org/ratelimit" "github.com/projectdiscovery/gologger/levels" @@ -60,41 +59,6 @@ var DefaultOptions = &types.Options{ CustomHeaders: []string{}, } -// MockOutputWriter is a mocked output writer. -type MockOutputWriter struct { - aurora aurora.Aurora - RequestCallback func(templateID, url, requestType string, err error) - WriteCallback func(o *output.ResultEvent) -} - -// NewMockOutputWriter creates a new mock output writer -func NewMockOutputWriter() *MockOutputWriter { - return &MockOutputWriter{aurora: aurora.NewAurora(false)} -} - -// Close closes the output writer interface -func (m *MockOutputWriter) Close() {} - -// Colorizer returns the colorizer instance for writer -func (m *MockOutputWriter) Colorizer() aurora.Aurora { - return m.aurora -} - -// Write writes the event to file and/or screen. -func (m *MockOutputWriter) Write(result *output.ResultEvent) error { - if m.WriteCallback != nil { - m.WriteCallback(result) - } - return nil -} - -// Request writes a log the requests trace log -func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) { - if m.RequestCallback != nil { - m.RequestCallback(templateID, url, requestType, err) - } -} - // TemplateInfo contains info for a mock executed template. type TemplateInfo struct { ID string @@ -109,7 +73,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco TemplateID: info.ID, TemplateInfo: info.Info, TemplatePath: info.Path, - Output: NewMockOutputWriter(), + Output: output.NewMockOutputWriter(), Options: options, Progress: progressImpl, ProjectFile: nil, diff --git a/v2/pkg/core/inputs/inputs.go b/v2/pkg/core/inputs/inputs.go new file mode 100644 index 00000000..6237dfb9 --- /dev/null +++ b/v2/pkg/core/inputs/inputs.go @@ -0,0 +1,17 @@ +package inputs + +type SimpleInputProvider struct { + Inputs []string +} + +// Count returns the number of items for input provider +func (s *SimpleInputProvider) Count() int64 { + return int64(len(s.Inputs)) +} + +// Scan calls a callback function till the input provider is exhausted +func (s *SimpleInputProvider) Scan(callback func(value string)) { + for _, v := range s.Inputs { + callback(v) + } +} diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index b90be29c..26555719 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -209,3 +209,38 @@ func (w *StandardWriter) Close() { w.traceFile.Close() } } + +// MockOutputWriter is a mocked output writer. +type MockOutputWriter struct { + aurora aurora.Aurora + RequestCallback func(templateID, url, requestType string, err error) + WriteCallback func(o *ResultEvent) +} + +// NewMockOutputWriter creates a new mock output writer +func NewMockOutputWriter() *MockOutputWriter { + return &MockOutputWriter{aurora: aurora.NewAurora(false)} +} + +// Close closes the output writer interface +func (m *MockOutputWriter) Close() {} + +// Colorizer returns the colorizer instance for writer +func (m *MockOutputWriter) Colorizer() aurora.Aurora { + return m.aurora +} + +// Write writes the event to file and/or screen. +func (m *MockOutputWriter) Write(result *ResultEvent) error { + if m.WriteCallback != nil { + m.WriteCallback(result) + } + return nil +} + +// Request writes a log the requests trace log +func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) { + if m.RequestCallback != nil { + m.RequestCallback(templateID, url, requestType, err) + } +} diff --git a/v2/pkg/progress/progress.go b/v2/pkg/progress/progress.go index 220ff49a..fcd34340 100644 --- a/v2/pkg/progress/progress.go +++ b/v2/pkg/progress/progress.go @@ -247,3 +247,27 @@ func (p *StatsTicker) Stop() { _ = p.server.Shutdown(context.Background()) } } + +type MockProgressClient struct{} + +// Stop stops the progress recorder. +func (m *MockProgressClient) Stop() {} + +// Init inits the progress bar with initial details for scan +func (m *MockProgressClient) Init(hostCount int64, rulesCount int, requestCount int64) {} + +// AddToTotal adds a value to the total request count +func (m *MockProgressClient) AddToTotal(delta int64) {} + +// IncrementRequests increments the requests counter by 1. +func (m *MockProgressClient) IncrementRequests() {} + +// IncrementMatched increments the matched counter by 1. +func (m *MockProgressClient) IncrementMatched() {} + +// IncrementErrorsBy increments the error counter by count. +func (m *MockProgressClient) IncrementErrorsBy(count int64) {} + +// IncrementFailedRequestsBy increments the number of requests counter by count +// along with errors. +func (m *MockProgressClient) IncrementFailedRequestsBy(count int64) {} diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 5c2914ff..0ea42d46 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -103,6 +103,20 @@ func New(options *Options) (*Client, error) { return interactClient, nil } +// NewDefaultOptions returns the default options for interactsh client +func NewDefaultOptions(output output.Writer, reporting *reporting.Client, progress progress.Progress) *Options { + return &Options{ + ServerURL: "https://interactsh.com", + CacheSize: 5000, + Eviction: 60 * time.Second, + ColldownPeriod: 5 * time.Second, + PollDuration: 5 * time.Second, + Output: output, + IssuesClient: reporting, + Progress: progress, + } +} + func (c *Client) firstTimeInitializeClient() error { interactsh, err := client.New(&client.Options{ ServerURL: c.options.ServerURL, diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 9f8f784b..7f5a1851 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -181,3 +181,17 @@ func (options *Options) AddVarPayload(key string, value interface{}) { func (options *Options) VarsPayload() map[string]interface{} { return options.varsPayload } + +// DefaultOptions returns default options for nuclei +func DefaultOptions() *Options { + return &Options{ + RateLimit: 150, + BulkSize: 25, + TemplateThreads: 25, + HeadlessBulkSize: 10, + HeadlessTemplateThreads: 10, + Timeout: 5, + Retries: 1, + MaxHostError: 30, + } +} From 75f18f169ca35be5ea603599a662d4ec7b2d4510 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 29 Oct 2021 18:30:24 +0530 Subject: [PATCH 054/146] Adding response highlighting to websocket --- v2/pkg/protocols/others/ssl/ssl.go | 2 +- v2/pkg/protocols/others/websocket/websocket.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/v2/pkg/protocols/others/ssl/ssl.go b/v2/pkg/protocols/others/ssl/ssl.go index ec72fcc9..ec6a32a1 100644 --- a/v2/pkg/protocols/others/ssl/ssl.go +++ b/v2/pkg/protocols/others/ssl/ssl.go @@ -96,7 +96,7 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu data["not_after"] = float64(cert.NotAfter.Unix()) data["ip"] = r.dialer.GetDialedIP(hostname) - event := eventcreator.CreateEventWithAdditionalOptions(r, data, r.options.Options.Debug || r.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {}) + event := eventcreator.CreateEvent(r, data, r.options.Options.Debug || r.options.Options.DebugResponse) callback(event) return nil } diff --git a/v2/pkg/protocols/others/websocket/websocket.go b/v2/pkg/protocols/others/websocket/websocket.go index a2487aa0..7c855854 100644 --- a/v2/pkg/protocols/others/websocket/websocket.go +++ b/v2/pkg/protocols/others/websocket/websocket.go @@ -22,6 +22,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -278,12 +279,6 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu r.options.Output.Request(r.options.TemplateID, input, "websocket", err) gologger.Verbose().Msgf("Sent Websocket request to %s", input) - if r.options.Options.Debug || r.options.Options.DebugResponse { - responseOutput := responseBuilder.String() - gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", r.options.TemplateID, input) - gologger.Print().Msgf("%s", responseOutput) - } - data := make(map[string]interface{}) for k, v := range previous { data[k] = v @@ -303,6 +298,11 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu event := eventcreator.CreateEventWithAdditionalOptions(r, data, r.options.Options.Debug || r.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { internalWrappedEvent.OperatorsResult.PayloadValues = payloadValues }) + if r.options.Options.Debug || r.options.Options.DebugResponse { + responseOutput := responseBuilder.String() + gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", r.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, r.options.Options.NoColor)) + } callback(event) return nil From 3a47413cd484eb5173d821981009fbd4c7000c2d Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Fri, 29 Oct 2021 19:08:18 +0300 Subject: [PATCH 055/146] Add case-insensitive flag to matchers and extractors --- v2/pkg/operators/extractors/compile.go | 10 ++++++++++ v2/pkg/operators/extractors/extract.go | 12 +++++++++++- v2/pkg/operators/extractors/extractors.go | 7 +++++++ v2/pkg/operators/matchers/compile.go | 10 ++++++++++ v2/pkg/operators/matchers/match.go | 4 ++++ v2/pkg/operators/matchers/matchers.go | 6 ++++++ 6 files changed, 48 insertions(+), 1 deletion(-) diff --git a/v2/pkg/operators/extractors/compile.go b/v2/pkg/operators/extractors/compile.go index e688c952..07306e20 100644 --- a/v2/pkg/operators/extractors/compile.go +++ b/v2/pkg/operators/extractors/compile.go @@ -46,5 +46,15 @@ func (e *Extractor) CompileExtractors() error { if e.Part == "" { e.Part = "body" } + + if e.CaseInsensitive { + if e.Type != "kval" { + return fmt.Errorf("case-insensitive flag is supported only for 'kval' extractors (not '%s')", e.Type) + } + for i := range e.KVal { + e.KVal[i] = strings.ToLower(e.KVal[i]) + } + } + return nil } diff --git a/v2/pkg/operators/extractors/extract.go b/v2/pkg/operators/extractors/extract.go index 72440c6e..e43ea145 100644 --- a/v2/pkg/operators/extractors/extract.go +++ b/v2/pkg/operators/extractors/extract.go @@ -34,8 +34,18 @@ func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} { // ExtractKval extracts key value pairs from a data map func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{} { - results := make(map[string]struct{}) + if e.CaseInsensitive { + inputData := data + data = make(map[string]interface{}, len(inputData)) + for k, v := range inputData { + if s, ok := v.(string); ok { + v = strings.ToLower(s) + } + data[strings.ToLower(k)] = v + } + } + results := make(map[string]struct{}) for _, k := range e.KVal { item, ok := data[k] if !ok { diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index 5c126a27..d4b2bb3c 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -105,6 +105,13 @@ type Extractor struct { // Internal, when set to true will allow using the value extracted // in the next request for some protocols (like HTTP). Internal bool `yaml:"internal,omitempty" jsonschema:"title=mark extracted value for internal variable use,description=Internal when set to true will allow using the value extracted in the next request for some protocols"` + + // description: | + // CaseInsensitive enables case-insensitive extractions. Default is false. + // values: + // - false + // - true + CaseInsensitive bool `yaml:"case-insensitive,omitempty" jsonschema:"title=use case insensitive extract,description=use case insensitive extract"` } // ExtractorType is the type of the extractor specified diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index 09213e23..63b3edf5 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "fmt" "regexp" + "strings" "github.com/Knetic/govaluate" @@ -60,5 +61,14 @@ func (m *Matcher) CompileMatchers() error { } else { m.condition = ORCondition } + + if m.CaseInsensitive { + if m.Type != "word" { + return fmt.Errorf("case-insensitive flag is supported only for 'word' matchers (not '%s')", m.Type) + } + for i := range m.Words { + m.Words[i] = strings.ToLower(m.Words[i]) + } + } return nil } diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index fff0055d..df71a144 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -42,6 +42,10 @@ func (m *Matcher) MatchSize(length int) bool { // MatchWords matches a word check against a corpus. func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) { + if m.CaseInsensitive { + corpus = strings.ToLower(corpus) + } + var matchedWords []string // Iterate over all the words accepted as valid for i, word := range m.Words { diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index 88ee413e..594a6fc1 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -105,6 +105,12 @@ type Matcher struct { // values: // - "hex" Encoding string `yaml:"encoding,omitempty" jsonschema:"title=encoding for word field,description=Optional encoding for the word fields,enum=hex"` + // description: | + // CaseInsensitive enables case-insensitive matches. Default is false. + // values: + // - false + // - true + CaseInsensitive bool `yaml:"case-insensitive,omitempty" jsonschema:"title=use case insensitive match,description=use case insensitive match"` // cached data for the compiled matcher condition ConditionType From bfb69b2ff2fca56e354598c03c91f5044ed658d9 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Fri, 29 Oct 2021 19:10:17 +0300 Subject: [PATCH 056/146] Extract getMatchPart method in protocols --- v2/pkg/protocols/dns/operators.go | 37 +++++++++++++------------- v2/pkg/protocols/file/operators.go | 33 ++++++++++++----------- v2/pkg/protocols/headless/operators.go | 33 ++++++++++++----------- v2/pkg/protocols/http/operators.go | 6 ++--- v2/pkg/protocols/network/operators.go | 33 ++++++++++++----------- 5 files changed, 73 insertions(+), 69 deletions(-) diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index b9f0454b..0cf54988 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -16,13 +16,7 @@ import ( // Match matches a generic data response again a given matcher func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - partString := matcher.Part - switch partString { - case "body", "all", "": - partString = "raw" - } - - item, ok := data[partString] + item, ok := request.getMatchPart(matcher.Part, data) if !ok { return false, []string{} } @@ -50,25 +44,32 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat // Extract performs extracting operation for an extractor on model and returns true or false. func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - part := extractor.Part + item, ok := request.getMatchPart(extractor.Part, data) + if !ok { + return nil + } + + switch extractor.GetType() { + case extractors.RegexExtractor: + return extractor.ExtractRegex(types.ToString(item)) + case extractors.KValExtractor: + return extractor.ExtractKval(data) + } + return nil +} + +func (request *Request) getMatchPart(part string, data output.InternalEvent) (interface{}, bool) { switch part { - case "body", "all": + case "body", "all", "": part = "raw" } item, ok := data[part] if !ok { - return nil + return "", false } - itemStr := types.ToString(item) - switch extractor.GetType() { - case extractors.RegexExtractor: - return extractor.ExtractRegex(itemStr) - case extractors.KValExtractor: - return extractor.ExtractKval(data) - } - return nil + return item, true } // responseToDSLMap converts a DNS response to a map for use in DSL matching diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index c0d95c5b..cde2e63e 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -16,17 +16,10 @@ import ( // Match matches a generic data response again a given matcher func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - partString := matcher.Part - switch partString { - case "body", "all", "data", "": - partString = "raw" - } - - item, ok := data[partString] + itemStr, ok := request.getMatchPart(matcher.Part, data) if !ok { return false, []string{} } - itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: @@ -45,17 +38,10 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat // Extract performs extracting operation for an extractor on model and returns true or false. func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - partString := extractor.Part - switch partString { - case "body", "all", "data", "": - partString = "raw" - } - - item, ok := data[partString] + itemStr, ok := request.getMatchPart(extractor.Part, data) if !ok { return nil } - itemStr := types.ToString(item) switch extractor.GetType() { case extractors.RegexExtractor: @@ -66,6 +52,21 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto return nil } +func (request *Request) getMatchPart(part string, data output.InternalEvent) (string, bool) { + switch part { + case "body", "all", "data", "": + part = "raw" + } + + item, ok := data[part] + if !ok { + return "", false + } + itemStr := types.ToString(item) + + return itemStr, true +} + // responseToDSLMap converts a file response to a map for use in DSL matching func (request *Request) responseToDSLMap(raw, inputFilePath, matchedFileName string) output.InternalEvent { return output.InternalEvent{ diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index 81a81a6a..3d62e61b 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -14,17 +14,10 @@ import ( // Match matches a generic data response again a given matcher func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - partString := matcher.Part - switch partString { - case "body", "resp", "": - partString = "data" - } - - item, ok := data[partString] + itemStr, ok := request.getMatchPart(matcher.Part, data) if !ok { return false, []string{} } - itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: @@ -43,17 +36,10 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat // Extract performs extracting operation for an extractor on model and returns true or false. func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - partString := extractor.Part - switch partString { - case "body", "resp", "": - partString = "data" - } - - item, ok := data[partString] + itemStr, ok := request.getMatchPart(extractor.Part, data) if !ok { return nil } - itemStr := types.ToString(item) switch extractor.GetType() { case extractors.RegexExtractor: @@ -64,6 +50,21 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto return nil } +func (request *Request) getMatchPart(part string, data output.InternalEvent) (string, bool) { + switch part { + case "body", "resp", "": + part = "data" + } + + item, ok := data[part] + if !ok { + return "", false + } + itemStr := types.ToString(item) + + return itemStr, true +} + // responseToDSLMap converts a headless response to a map for use in DSL matching func (request *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent { return output.InternalEvent{ diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 9ae939cf..8d7c4b72 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -17,7 +17,7 @@ import ( // Match matches a generic data response again a given matcher func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - item, ok := getMatchPart(matcher.Part, data) + item, ok := request.getMatchPart(matcher.Part, data) if !ok { return false, []string{} } @@ -57,7 +57,7 @@ func getStatusCode(data map[string]interface{}) (int, bool) { // Extract performs extracting operation for an extractor on model and returns true or false. func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - item, ok := getMatchPart(extractor.Part, data) + item, ok := request.getMatchPart(extractor.Part, data) if !ok { return nil } @@ -75,7 +75,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto } // getMatchPart returns the match part honoring "all" matchers + others. -func getMatchPart(part string, data output.InternalEvent) (string, bool) { +func (request *Request) getMatchPart(part string, data output.InternalEvent) (string, bool) { if part == "header" { part = "all_headers" } diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 8e0330dc..c829b8ad 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -14,17 +14,10 @@ import ( // Match matches a generic data response again a given matcher func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - partString := matcher.Part - switch partString { - case "body", "all", "": - partString = "data" - } - - item, ok := data[partString] + itemStr, ok := request.getMatchPart(matcher.Part, data) if !ok { return false, []string{} } - itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: @@ -43,17 +36,10 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat // Extract performs extracting operation for an extractor on model and returns true or false. func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - partString := extractor.Part - switch partString { - case "body", "all", "": - partString = "data" - } - - item, ok := data[partString] + itemStr, ok := request.getMatchPart(extractor.Part, data) if !ok { return nil } - itemStr := types.ToString(item) switch extractor.GetType() { case extractors.RegexExtractor: @@ -64,6 +50,21 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto return nil } +func (request *Request) getMatchPart(part string, data output.InternalEvent) (string, bool) { + switch part { + case "body", "all", "": + part = "data" + } + + item, ok := data[part] + if !ok { + return "", false + } + itemStr := types.ToString(item) + + return itemStr, true +} + // responseToDSLMap converts a network response to a map for use in DSL matching func (request *Request) responseToDSLMap(req, resp, raw, host, matched string) output.InternalEvent { return output.InternalEvent{ From 897f11213c16d6a45b4ff78befa74edc17cbd296 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Fri, 29 Oct 2021 19:11:09 +0300 Subject: [PATCH 057/146] Add unit tests for case-insensitive flag in protocols --- v2/pkg/protocols/dns/operators_test.go | 24 +++++++++++++++++ v2/pkg/protocols/file/operators_test.go | 20 ++++++++++++++ v2/pkg/protocols/http/operators_test.go | 31 ++++++++++++++++++++++ v2/pkg/protocols/network/operators_test.go | 19 +++++++++++++ 4 files changed, 94 insertions(+) diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index c8d06124..6bb91b7e 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -134,6 +134,30 @@ func TestDNSOperatorMatch(t *testing.T) { require.False(t, isMatched, "could match invalid response matcher") require.Equal(t, []string{}, matched) }) + + t.Run("caseInsensitive", func(t *testing.T) { + req := new(dns.Msg) + req.Question = append(req.Question, dns.Question{Name: "ONE.ONE.ONE.ONE.", Qtype: dns.TypeA, Qclass: dns.ClassINET}) + + resp := new(dns.Msg) + resp.Rcode = dns.RcodeSuccess + resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "ONE.ONE.ONE.ONE."}}) + + event := request.responseToDSLMap(req, resp, "ONE.ONE.ONE.ONE", "ONE.ONE.ONE.ONE") + + matcher := &matchers.Matcher{ + Part: "raw", + Type: "word", + Words: []string{"one.ONE.one.ONE"}, + CaseInsensitive: true, + } + err = matcher.CompileMatchers() + require.Nil(t, err, "could not compile matcher") + + isMatch, matched := request.Match(event, matcher) + require.True(t, isMatch, "could not match valid response") + require.Equal(t, []string{"one.one.one.one"}, matched) + }) } func TestDNSOperatorExtract(t *testing.T) { diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index b03d641b..da40c76d 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -105,6 +105,26 @@ func TestFileOperatorMatch(t *testing.T) { require.False(t, isMatched, "could match invalid response matcher") require.Equal(t, []string{}, matched) }) + + t.Run("caseInsensitive", func(t *testing.T) { + resp := "TEST-DATA\r\n1.1.1.1\r\n" + event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") + require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Equal(t, resp, event["raw"], "could not get correct resp") + + matcher := &matchers.Matcher{ + Part: "raw", + Type: "word", + Words: []string{"TeSt-DaTA"}, + CaseInsensitive: true, + } + err = matcher.CompileMatchers() + require.Nil(t, err, "could not compile matcher") + + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, []string{"test-data"}, matched) + }) } func TestFileOperatorExtract(t *testing.T) { diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index c9d3b795..e2ecbdf5 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -117,6 +117,21 @@ func TestHTTPOperatorMatch(t *testing.T) { require.False(t, isMatched, "could match invalid response matcher") require.Equal(t, []string{}, matched) }) + + t.Run("caseInsensitive", func(t *testing.T) { + matcher := &matchers.Matcher{ + Part: "body", + Type: "word", // only applies to word + Words: []string{"EXAMPLE DOMAIN"}, + CaseInsensitive: true, + } + err = matcher.CompileMatchers() + require.Nil(t, err, "could not compile matcher") + + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, []string{"example domain"}, matched) + }) } func TestHTTPOperatorExtract(t *testing.T) { @@ -215,6 +230,22 @@ func TestHTTPOperatorExtract(t *testing.T) { require.Equal(t, map[string]struct{}{"{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]}": {}}, data, "could not extract correct json data") }) }) + + t.Run("caseInsensitive", func(t *testing.T) { + event["body"] = exampleResponseBody + + extractor := &extractors.Extractor{ + Type: "kval", + KVal: []string{"TEST_HEADER"}, // only applies to KVal + CaseInsensitive: true, + } + err = extractor.CompileExtractors() + require.Nil(t, err, "could not compile kval extractor") + + data := request.Extract(event, extractor) + require.Greater(t, len(data), 0, "could not extractor kval valid response") + require.Equal(t, map[string]struct{}{"test-response": {}}, data, "could not extract correct kval data") + }) } func TestHTTPMakeResult(t *testing.T) { diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index bf577d7e..735d9af3 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -103,6 +103,25 @@ func TestNetworkOperatorMatch(t *testing.T) { require.False(t, isMatched, "could match invalid response matcher") require.Equal(t, []string{}, matched) }) + + t.Run("caseInsensitive", func(t *testing.T) { + matcher := &matchers.Matcher{ + Part: "body", + Type: "word", + Words: []string{"rESp-DAta"}, + CaseInsensitive: true, + } + err = matcher.CompileMatchers() + require.Nil(t, err, "could not compile matcher") + + req := "TEST-DATA\r\n" + resp := "RESP-DATA\r\nSTAT \r\n" + event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", "TEST") + + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, []string{"resp-data"}, matched) + }) } func TestNetworkOperatorExtract(t *testing.T) { From 392ea23f1d0b97d95492ed173b9958fa11a56d62 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Fri, 29 Oct 2021 19:11:57 +0300 Subject: [PATCH 058/146] Add integration tests for case-insensitive flag --- .../http/get-case-insensitive.yaml | 16 ++++ v2/cmd/integration-test/http.go | 82 +++++++++++++++---- 2 files changed, 80 insertions(+), 18 deletions(-) create mode 100644 integration_tests/http/get-case-insensitive.yaml diff --git a/integration_tests/http/get-case-insensitive.yaml b/integration_tests/http/get-case-insensitive.yaml new file mode 100644 index 00000000..e8c4054b --- /dev/null +++ b/integration_tests/http/get-case-insensitive.yaml @@ -0,0 +1,16 @@ +id: basic-get-case-insensitive + +info: + name: Basic GET Request + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + matchers: + - type: word + case-insensitive: true + words: + - "ThIS is TEsT MAtcHEr TExT" diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index c8bc6acc..a81111c4 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -15,24 +15,26 @@ import ( ) var httpTestcases = map[string]testutils.TestCase{ - "http/get-headers.yaml": &httpGetHeaders{}, - "http/get-query-string.yaml": &httpGetQueryString{}, - "http/get-redirects.yaml": &httpGetRedirects{}, - "http/get.yaml": &httpGet{}, - "http/post-body.yaml": &httpPostBody{}, - "http/post-json-body.yaml": &httpPostJSONBody{}, - "http/post-multipart-body.yaml": &httpPostMultipartBody{}, - "http/raw-cookie-reuse.yaml": &httpRawCookieReuse{}, - "http/raw-dynamic-extractor.yaml": &httpRawDynamicExtractor{}, - "http/raw-get-query.yaml": &httpRawGetQuery{}, - "http/raw-get.yaml": &httpRawGet{}, - "http/raw-payload.yaml": &httpRawPayload{}, - "http/raw-post-body.yaml": &httpRawPostBody{}, - "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{}, - "http/request-condition.yaml": &httpRequestCondition{}, - "http/request-condition-new.yaml": &httpRequestCondition{}, - "http/interactsh.yaml": &httpInteractshRequest{}, - "http/self-contained.yaml": &httpRequestSelContained{}, + "http/get-headers.yaml": &httpGetHeaders{}, + "http/get-query-string.yaml": &httpGetQueryString{}, + "http/get-redirects.yaml": &httpGetRedirects{}, + "http/get.yaml": &httpGet{}, + "http/post-body.yaml": &httpPostBody{}, + "http/post-json-body.yaml": &httpPostJSONBody{}, + "http/post-multipart-body.yaml": &httpPostMultipartBody{}, + "http/raw-cookie-reuse.yaml": &httpRawCookieReuse{}, + "http/raw-dynamic-extractor.yaml": &httpRawDynamicExtractor{}, + "http/raw-get-query.yaml": &httpRawGetQuery{}, + "http/raw-get.yaml": &httpRawGet{}, + "http/raw-payload.yaml": &httpRawPayload{}, + "http/raw-post-body.yaml": &httpRawPostBody{}, + "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{}, + "http/request-condition.yaml": &httpRequestCondition{}, + "http/request-condition-new.yaml": &httpRequestCondition{}, + "http/interactsh.yaml": &httpInteractshRequest{}, + "http/self-contained.yaml": &httpRequestSelContained{}, + "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{}, + "http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{}, } type httpInteractshRequest struct{} @@ -553,3 +555,47 @@ func (h *httpRequestSelContained) Execute(filePath string) error { } return nil } + +type httpGetCaseInsensitive struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpGetCaseInsensitive) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "THIS IS TEST MATCHER TEXT") + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + +type httpGetCaseInsensitiveCluster struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpGetCaseInsensitiveCluster) Execute(filesPath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "This is test matcher text") + }) + ts := httptest.NewServer(router) + defer ts.Close() + + files := strings.Split(filesPath, ",") + + results, err := testutils.RunNucleiTemplateAndGetResults(files[0], ts.URL, debug, "-t", files[1]) + if err != nil { + return err + } + if len(results) != 2 { + return errIncorrectResultsCount(results) + } + return nil +} From 04e3c0165a31965bbafd46f3a4e2ec8c61680fd3 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Sat, 30 Oct 2021 13:17:47 +0300 Subject: [PATCH 059/146] feat: In case of binary data, show a hexadecimal view as well #1080 --- v2/cmd/nuclei/main.go | 7 -- .../helpers/responsehighlighter/hexdump.go | 116 ++++++++++++++++++ .../response_highlighter.go | 17 ++- .../response_highlighter_test.go | 78 ++++++++++++ v2/pkg/protocols/dns/request.go | 27 ++-- v2/pkg/protocols/file/request.go | 23 +++- v2/pkg/protocols/headless/request.go | 34 ++--- v2/pkg/protocols/http/request.go | 35 +++++- v2/pkg/protocols/network/request.go | 16 ++- 9 files changed, 303 insertions(+), 50 deletions(-) create mode 100644 v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go create mode 100644 v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index d2f8ebb9..32e6221f 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -175,10 +175,3 @@ func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags currentFlag.Group(groupName) } } - -/* -HacktoberFest update: Below, you can find our ticket recommendations. Tasks with the "good first issue" label are suitable for first time contributors. If you have other ideas, or need help with getting started, join our Discord channel or reach out to @forgedhallpass. - -https://github.com/issues?q=is%3Aopen+is%3Aissue+user%3Aprojectdiscovery+label%3AHacktoberfest - -*/ diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go b/v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go new file mode 100644 index 00000000..10c2cfc5 --- /dev/null +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go @@ -0,0 +1,116 @@ +package responsehighlighter + +import ( + "errors" + "fmt" + "regexp" + "strings" + "unicode" + + "github.com/projectdiscovery/gologger" +) + +// [0-9a-fA-F]{8} {2} - hexdump indexes (8 character hex value followed by two spaces) +// [0-9a-fA-F]{2} + - 2 character long hex values followed by one or two space (potentially wrapped with an ASCII color code, see below) +// \x1b\[(\d;?)+m - ASCII color code pattern +// \x1b\[0m - ASCII color code reset +// \|(.*)\|\n - ASCII representation of the input delimited by pipe characters +var hexDumpParsePattern = regexp.MustCompile(`([0-9a-fA-F]{8} {2})((?:(?:\x1b\[(?:\d;?)+m)?[0-9a-fA-F]{2}(?:\x1b\[0m)? +)+)\|(.*)\|\n`) +var hexValuePattern = regexp.MustCompile(`([a-fA-F0-9]{2})`) + +type HighlightableHexDump struct { + index []string + hex []string + ascii []string +} + +func NewHighlightableHexDump(rowSize int) HighlightableHexDump { + return HighlightableHexDump{index: make([]string, 0, rowSize), hex: make([]string, 0, rowSize), ascii: make([]string, 0, rowSize)} +} + +func (hexDump HighlightableHexDump) len() int { + return len(hexDump.index) +} + +func (hexDump HighlightableHexDump) String() string { + var result string + for i := 0; i < hexDump.len(); i++ { + result += hexDump.index[i] + hexDump.hex[i] + "|" + hexDump.ascii[i] + "|\n" + } + return result +} + +func toHighLightedHexDump(hexDump, snippetToHighlight string) (HighlightableHexDump, error) { + hexDumpRowValues := hexDumpParsePattern.FindAllStringSubmatch(hexDump, -1) + if hexDumpRowValues == nil || len(hexDumpRowValues) != strings.Count(hexDump, "\n") { + message := "could not parse hexdump" + gologger.Warning().Msgf(message) + return HighlightableHexDump{}, errors.New(message) + } + + result := NewHighlightableHexDump(len(hexDumpRowValues)) + for _, currentHexDumpRowValues := range hexDumpRowValues { + result.index = append(result.index, currentHexDumpRowValues[1]) + result.hex = append(result.hex, currentHexDumpRowValues[2]) + result.ascii = append(result.ascii, currentHexDumpRowValues[3]) + } + return result.highlight(snippetToHighlight), nil +} + +func (hexDump HighlightableHexDump) highlight(snippetToColor string) HighlightableHexDump { + return highlightAsciiSection(highlightHexSection(hexDump, snippetToColor), snippetToColor) +} + +func highlightHexSection(hexDump HighlightableHexDump, snippetToColor string) HighlightableHexDump { + var snippetHexCharactersMatchPattern string + for _, char := range snippetToColor { + snippetHexCharactersMatchPattern += fmt.Sprintf(`(%02x[ \n]+)`, char) + } + + hexDump.hex = highlight(hexDump.hex, snippetHexCharactersMatchPattern, func(v string) string { + return hexValuePattern.ReplaceAllString(v, addColor("$1")) + }) + + return hexDump +} + +func highlightAsciiSection(hexDump HighlightableHexDump, snippetToColor string) HighlightableHexDump { + var snippetCharactersMatchPattern string + for _, v := range snippetToColor { + snippetCharactersMatchPattern += fmt.Sprintf(`(%s\n*)`, regexp.QuoteMeta(string(v))) + } + + hexDump.ascii = highlight(hexDump.ascii, snippetCharactersMatchPattern, func(v string) string { + if len(v) > 1 { + return addColor(string(v[0])) + v[1:] // do not color new line characters + } + return addColor(v) + }) + + return hexDump +} + +func highlight(values []string, snippetCharactersMatchPattern string, replaceToFunc func(v string) string) []string { + rows := strings.Join(values, "\n") + compiledPattern := regexp.MustCompile(snippetCharactersMatchPattern) + for _, submatch := range compiledPattern.FindAllStringSubmatch(rows, -1) { + var replaceTo string + var replaceFrom string + for _, matchedValueWithSuffix := range submatch[1:] { + replaceFrom += matchedValueWithSuffix + replaceTo += replaceToFunc(matchedValueWithSuffix) + } + rows = strings.ReplaceAll(rows, replaceFrom, replaceTo) + } + return strings.Split(rows, "\n") +} + +// IsASCII tests whether a string consists only of ASCII characters or not +func IsASCII(input string) bool { + for i := 0; i < len(input); i++ { + if input[i] > unicode.MaxASCII { + return false + } + } + return true +} diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go index 1cb914e9..f637644f 100644 --- a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go @@ -9,15 +9,22 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" ) -var colorizer = aurora.NewAurora(true) +var colorFunction = aurora.Green -func Highlight(operatorResult *operators.Result, response string, noColor bool) string { +func Highlight(operatorResult *operators.Result, response string, noColor, hexDump bool) string { result := response if operatorResult != nil && !noColor { for _, matches := range operatorResult.Matches { if len(matches) > 0 { for _, currentMatch := range matches { - result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + if hexDump { + highlightedHexDump, err := toHighLightedHexDump(result, currentMatch) + if err == nil { + result = highlightedHexDump.String() + } + } else { + result = strings.ReplaceAll(result, currentMatch, addColor(currentMatch)) + } } } } @@ -33,3 +40,7 @@ func CreateStatusCodeSnippet(response string, statusCode int) string { } return "" } + +func addColor(value string) string { + return colorFunction(value).String() +} diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go new file mode 100644 index 00000000..a60341ef --- /dev/null +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go @@ -0,0 +1,78 @@ +package responsehighlighter + +import ( + "encoding/hex" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/projectdiscovery/nuclei/v2/pkg/operators" +) + +const input = "abcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmn" + +func TestHexDumpHighlighting(t *testing.T) { + const highlightedHexDumpResponse = `00000000 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 |abcdefghijklmnab| +00000010 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 |cdefghijklmnabcd| +00000020 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 |efghijklmnabcdef| +00000030 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 |ghijklmnabcdefgh| +00000040 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a |ijklmnabcdefghij| +00000050 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c |klmnabcdefghijkl| +00000060 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |mnabcdefghijklmn| +00000070 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |abcdefghijklmn| +` + t.Run("Test highlighting when the snippet is wrapped", func(t *testing.T) { + result, err := toHighLightedHexDump(hex.Dump([]byte(input)), "defghij") + assert.Nil(t, err) + assert.Equal(t, highlightedHexDumpResponse, result.String()) + }) + + t.Run("Test highlight when the snippet contains separator character", func(t *testing.T) { + value := "asdfasdfasda|basdfadsdfs|" + result, err := toHighLightedHexDump(hex.Dump([]byte(value)), "a|b") + + expected := `00000000 61 73 64 66 61 73 64 66 61 73 64 61 7c 62 61 73 |asdfasdfasda|bas| +00000010 64 66 61 64 73 64 66 73 7c |dfadsdfs|| +` + assert.Nil(t, err) + assert.Equal(t, expected, result.String()) + }) +} + +func TestHighlight(t *testing.T) { + const multiSnippetHighlightHexDumpResponse = `00000000 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 |abcdefghijklmnab| +00000010 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 |cdefghijklmnabcd| +00000020 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 |efghijklmnabcdef| +00000030 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 |ghijklmnabcdefgh| +00000040 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a |ijklmnabcdefghij| +00000050 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c |klmnabcdefghijkl| +00000060 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |mnabcdefghijklmn| +00000070 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |abcdefghijklmn| +` + matches := map[string][]string{ + "first": {"defghij"}, + "second": {"ab"}, + } + operatorResult := operators.Result{Matches: matches} + + t.Run("Test highlighting when the snippet is wrapped", func(t *testing.T) { + result := Highlight(&operatorResult, hex.Dump([]byte(input)), false, true) + assert.Equal(t, multiSnippetHighlightHexDumpResponse, result) + }) + + t.Run("Test highlighting without hexdump", func(t *testing.T) { + result := Highlight(&operatorResult, input, false, false) + expected := `abcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmn` + assert.Equal(t, expected, result) + }) + + t.Run("Test the response is not modified if noColor is true", func(t *testing.T) { + result := Highlight(&operatorResult, input, true, false) + assert.Equal(t, input, result) + }) + + t.Run("Test the response is not modified if noColor is true", func(t *testing.T) { + result := Highlight(&operatorResult, hex.Dump([]byte(input)), true, true) + assert.Equal(t, hex.Dump([]byte(input)), result) + }) +} diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 28a3309b..61927ea4 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -1,6 +1,7 @@ package dns import ( + "encoding/hex" "net/url" "github.com/pkg/errors" @@ -44,35 +45,45 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } // Send the request to the target servers - resp, err := request.dnsClient.Do(compiledRequest) + response, err := request.dnsClient.Do(compiledRequest) if err != nil { request.options.Output.Request(request.options.TemplateID, domain, "dns", err) request.options.Progress.IncrementFailedRequestsBy(1) } - if resp == nil { + if response == nil { return errors.Wrap(err, "could not send dns request") } request.options.Progress.IncrementRequests() request.options.Output.Request(request.options.TemplateID, domain, "dns", err) - gologger.Verbose().Msgf("[%s] Sent DNS request to %s", request.options.TemplateID, domain) + gologger.Verbose().Msgf("[%s] Sent DNS request to %s\n", request.options.TemplateID, domain) - outputEvent := request.responseToDSLMap(compiledRequest, resp, input, input) + outputEvent := request.responseToDSLMap(compiledRequest, response, input, input) for k, v := range previous { outputEvent[k] = v } event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, resp.String(), request.options.Options.NoColor)) - } + debug(event, request, domain, response.String()) callback(event) return nil } +func debug(event *output.InternalWrappedEvent, request *Request, domain string, response string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped DNS response for %s\n", request.options.TemplateID, domain) + + hexDump := false + if !responsehighlighter.IsASCII(response) { + hexDump = true + response = hex.Dump([]byte(response)) + } + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump)) + } +} + // isURL tests a string to determine if it is a well-structured url or not. func isURL(toTest string) bool { if _, err := url.ParseRequestURI(toTest); err != nil { diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index 61bb8718..e27331ed 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -1,6 +1,8 @@ package file import ( + "encoding/hex" + "fmt" "io/ioutil" "os" @@ -49,20 +51,17 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review gologger.Error().Msgf("Could not read file path %s: %s\n", filePath, err) return } - dataStr := tostring.UnsafeToString(buffer) + fileContent := tostring.UnsafeToString(buffer) gologger.Verbose().Msgf("[%s] Sent FILE request to %s", request.options.TemplateID, filePath) - outputEvent := request.responseToDSLMap(dataStr, input, filePath) + outputEvent := request.responseToDSLMap(fileContent, input, filePath) for k, v := range previous { outputEvent[k] = v } event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, dataStr, request.options.Options.NoColor)) - } + debug(event, request, filePath, fileContent) callback(event) }(data) @@ -76,3 +75,15 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review request.options.Progress.IncrementRequests() return nil } + +func debug(event *output.InternalWrappedEvent, request *Request, filePath string, fileContent string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + hexDump := false + if !responsehighlighter.IsASCII(fileContent) { + hexDump = true + fileContent = hex.Dump([]byte(fileContent)) + } + logHeader := fmt.Sprintf("[%s] Dumped file request for %s\n", request.options.TemplateID, filePath) + gologger.Debug().Msgf("%s\n%s", logHeader, responsehighlighter.Highlight(event.OperatorsResult, fileContent, request.options.Options.NoColor, hexDump)) + } +} diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 55394ed7..0486009a 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -17,42 +17,42 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (request *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(inputURL string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { instance, err := request.options.Browser.NewInstance() if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Output.Request(request.options.TemplateID, inputURL, "headless", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } defer instance.Close() - parsed, err := url.Parse(input) + parsedURL, err := url.Parse(inputURL) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Output.Request(request.options.TemplateID, inputURL, "headless", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } - out, page, err := instance.Run(parsed, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second) + out, page, err := instance.Run(parsedURL, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Output.Request(request.options.TemplateID, inputURL, "headless", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } defer page.Close() - request.options.Output.Request(request.options.TemplateID, input, "headless", nil) + request.options.Output.Request(request.options.TemplateID, inputURL, "headless", nil) request.options.Progress.IncrementRequests() - gologger.Verbose().Msgf("Sent Headless request to %s", input) + gologger.Verbose().Msgf("Sent Headless request to %s", inputURL) reqBuilder := &strings.Builder{} if request.options.Options.Debug || request.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, input) + gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, inputURL) for _, act := range request.Steps { reqBuilder.WriteString(act.String()) reqBuilder.WriteString("\n") } - gologger.Print().Msgf("%s", reqBuilder.String()) + gologger.Print().Msgf(reqBuilder.String()) } var responseBody string @@ -60,18 +60,22 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp if err == nil { responseBody, _ = html.HTML() } - outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), input, input) + outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL) for k, v := range out { outputEvent[k] = v } event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor)) - } + debug(event, request, responseBody, inputURL) callback(event) return nil } + +func debug(event *output.InternalWrappedEvent, request *Request, responseBody string, input string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n", request.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor, false)) + } +} diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 3451e5f3..da855883 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -2,6 +2,7 @@ package http import ( "bytes" + "encoding/hex" "fmt" "io" "io/ioutil" @@ -450,7 +451,8 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate // Decode gbk response content-types // gb18030 supersedes gb2312 - if isContentTypeGbk(resp.Header.Get("Content-Type")) { + responseContentType := resp.Header.Get("Content-Type") + if isContentTypeGbk(responseContentType) { dumpedResponse, err = decodegbk(dumpedResponse) if err != nil { return errors.Wrap(err, "could not gbk decode") @@ -509,10 +511,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta }) - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, string(redirectedResponse), request.options.Options.NoColor)) - } + debug(request, formedURL, redirectedResponse, responseContentType, event) callback(event) return nil @@ -532,3 +531,29 @@ func (request *Request) setCustomHeaders(req *generatedRequest) { } } } + +const CRLF = "\r\n" + +func debug(request *Request, formedURL string, redirectedResponse []byte, responseContentType string, event *output.InternalWrappedEvent) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + hexDump := false + response := string(redirectedResponse) + + var headers string + if responseContentType == "" || responseContentType == "application/octet-stream" || (responseContentType == "application/x-www-form-urlencoded" && responsehighlighter.IsASCII(response)) { + hexDump = true + responseLines := strings.Split(response, CRLF) + for i, value := range responseLines { + headers += value + CRLF + if value == "" { + response = hex.Dump([]byte(strings.Join(responseLines[i+1:], ""))) + break + } + } + } + logMessageHeader := fmt.Sprintf("[%s] Dumped HTTP response for %s\n", request.options.TemplateID, formedURL) + + gologger.Debug().Msgf("%s\n%s", logMessageHeader, responsehighlighter.Highlight(event.OperatorsResult, headers, request.options.Options.NoColor, false)) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump)) + } +} diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 66da31e9..12e34e8d 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -190,8 +190,8 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input if request.options.Options.Debug || request.options.Options.DebugRequests { requestOutput := reqBuilder.String() - gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", request.options.TemplateID, actualAddress) - gologger.Print().Msgf("%s\nHex: %s", requestOutput, hex.EncodeToString([]byte(requestOutput))) + gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n", request.options.TemplateID, actualAddress) + gologger.Print().Msgf("%s", hex.Dump([]byte(requestOutput))) } request.options.Output.Request(request.options.TemplateID, actualAddress, "network", err) @@ -274,14 +274,18 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input }) } - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Network response for %s", request.options.TemplateID, actualAddress) - gologger.Print().Msgf("%s\nHex: %s", response, responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), request.options.Options.NoColor)) - } + debug(event, request, response, actualAddress) return nil } +func debug(event *output.InternalWrappedEvent, request *Request, response string, actualAddress string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Network response for %s\n", request.options.TemplateID, actualAddress) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, hex.Dump([]byte(response)), request.options.Options.NoColor, true)) + } +} + // getAddress returns the address of the host to make request to func getAddress(toTest string) (string, error) { if strings.Contains(toTest, "://") { From 3f1186da2bfb740eec30e74ebcd2134aed0e1de6 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 30 Oct 2021 12:39:38 +0300 Subject: [PATCH 060/146] Add error log support --- v2/cmd/nuclei/main.go | 1 + v2/internal/runner/runner.go | 2 +- v2/pkg/output/output.go | 40 ++++++++++++++++++++++++++++-------- v2/pkg/types/types.go | 2 ++ 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index d2f8ebb9..da5a1ddb 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -139,6 +139,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"), flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"), flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), + flagSet.StringVarP(&options.ErrorLogFile, "error-log", "elog", "", "file to write sent requests error log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index b3f80695..ca74f6a4 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -215,7 +215,7 @@ func New(options *types.Options) (*Runner, error) { } // Create the output file if asked - outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.Output, options.TraceLogFile) + outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.Output, options.TraceLogFile, options.ErrorLogFile) if err != nil { return nil, errors.Wrap(err, "could not create output file") } diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index b90be29c..572045f5 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -41,6 +41,8 @@ type StandardWriter struct { outputMutex *sync.Mutex traceFile *fileWriter traceMutex *sync.Mutex + errorFile *fileWriter + errorMutex *sync.Mutex severityColors func(severity.Severity) string } @@ -97,7 +99,7 @@ type ResultEvent struct { } // NewStandardWriter creates a new output writer based on user configurations -func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, file, traceFile string) (*StandardWriter, error) { +func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, file, traceFile string, errorFile string) (*StandardWriter, error) { auroraColorizer := aurora.NewAurora(colors) var outputFile *fileWriter @@ -116,6 +118,14 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, } traceOutput = output } + var errorOutput *fileWriter + if errorFile != "" { + output, err := newFileOutputWriter(errorFile) + if err != nil { + return nil, errors.Wrap(err, "could not create error file") + } + errorOutput = output + } writer := &StandardWriter{ json: json, jsonReqResp: jsonReqResp, @@ -126,6 +136,8 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, outputMutex: &sync.Mutex{}, traceFile: traceOutput, traceMutex: &sync.Mutex{}, + errorFile: errorOutput, + errorMutex: &sync.Mutex{}, severityColors: colorizer.New(auroraColorizer), } return writer, nil @@ -171,8 +183,8 @@ type JSONTraceRequest struct { } // Request writes a log the requests trace log -func (w *StandardWriter) Request(templateID, url, requestType string, err error) { - if w.traceFile == nil { +func (w *StandardWriter) Request(templateID, url, requestType string, requestErr error) { + if w.traceFile == nil && w.errorFile == nil { return } request := &JSONTraceRequest{ @@ -180,8 +192,8 @@ func (w *StandardWriter) Request(templateID, url, requestType string, err error) URL: url, Type: requestType, } - if err != nil { - request.Error = err.Error() + if requestErr != nil { + request.Error = requestErr.Error() } else { request.Error = "none" } @@ -190,9 +202,18 @@ func (w *StandardWriter) Request(templateID, url, requestType string, err error) if err != nil { return } - w.traceMutex.Lock() - _ = w.traceFile.Write(data) - w.traceMutex.Unlock() + + if w.traceFile != nil { + w.traceMutex.Lock() + _ = w.traceFile.Write(data) + w.traceMutex.Unlock() + } + + if requestErr != nil && w.errorFile != nil { + w.errorMutex.Lock() + _ = w.errorFile.Write(data) + w.errorMutex.Unlock() + } } // Colorizer returns the colorizer instance for writer @@ -208,4 +229,7 @@ func (w *StandardWriter) Close() { if w.traceFile != nil { w.traceFile.Close() } + if w.errorFile != nil { + w.errorFile.Close() + } } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index d258f424..dcab9eae 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -57,6 +57,8 @@ type Options struct { TemplatesDirectory string // TraceLogFile specifies a file to write with the trace of all requests TraceLogFile string + // ErrorLogFile specifies a file to write with the errors of all requests + ErrorLogFile string // ReportingDB is the db for report storage as well as deduplication ReportingDB string // ReportingConfig is the config file for nuclei reporting module From 933ed2429d533f7d78ce30df4eee51ade0313be9 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 30 Oct 2021 12:46:26 +0300 Subject: [PATCH 061/146] Update json log request format --- v2/pkg/output/output.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 572045f5..57c841a8 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -174,23 +174,23 @@ func (w *StandardWriter) Write(event *ResultEvent) error { return nil } -// JSONTraceRequest is a trace log request written to file -type JSONTraceRequest struct { - ID string `json:"id"` - URL string `json:"url"` - Error string `json:"error"` - Type string `json:"type"` +// JSONLogRequest is a trace/error log request written to file +type JSONLogRequest struct { + Template string `json:"template"` + Input string `json:"input"` + Error string `json:"error"` + Type string `json:"type"` } // Request writes a log the requests trace log -func (w *StandardWriter) Request(templateID, url, requestType string, requestErr error) { +func (w *StandardWriter) Request(templatePath, input, requestType string, requestErr error) { if w.traceFile == nil && w.errorFile == nil { return } - request := &JSONTraceRequest{ - ID: templateID, - URL: url, - Type: requestType, + request := &JSONLogRequest{ + Template: templatePath, + Input: input, + Type: requestType, } if requestErr != nil { request.Error = requestErr.Error() From 463c1c0142968de19e97c9d18667acc1c74bbf71 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 30 Oct 2021 12:55:02 +0300 Subject: [PATCH 062/146] Use template path in output request --- v2/pkg/protocols/dns/request.go | 6 +++--- v2/pkg/protocols/file/request.go | 2 +- v2/pkg/protocols/headless/request.go | 8 ++++---- v2/pkg/protocols/http/request.go | 4 ++-- v2/pkg/protocols/network/request.go | 18 +++++++++--------- v2/pkg/protocols/offlinehttp/request.go | 2 +- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 28a3309b..8980aeb6 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -28,7 +28,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review // Compile each request for the template based on the URL compiledRequest, err := request.Make(domain) if err != nil { - request.options.Output.Request(request.options.TemplateID, domain, "dns", err) + request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not build request") } @@ -46,7 +46,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review // Send the request to the target servers resp, err := request.dnsClient.Do(compiledRequest) if err != nil { - request.options.Output.Request(request.options.TemplateID, domain, "dns", err) + request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) request.options.Progress.IncrementFailedRequestsBy(1) } if resp == nil { @@ -54,7 +54,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } request.options.Progress.IncrementRequests() - request.options.Output.Request(request.options.TemplateID, domain, "dns", err) + request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) gologger.Verbose().Msgf("[%s] Sent DNS request to %s", request.options.TemplateID, domain) outputEvent := request.responseToDSLMap(compiledRequest, resp, input, input) diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index 61bb8718..c1dfe4a2 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -69,7 +69,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review }) wg.Wait() if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "file", err) + request.options.Output.Request(request.options.TemplatePath, input, "file", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 55394ed7..35bdf28b 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -20,7 +20,7 @@ var _ protocols.Request = &Request{} func (request *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { instance, err := request.options.Browser.NewInstance() if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Output.Request(request.options.TemplatePath, input, "headless", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } @@ -28,19 +28,19 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp parsed, err := url.Parse(input) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Output.Request(request.options.TemplatePath, input, "headless", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } out, page, err := instance.Run(parsed, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Output.Request(request.options.TemplatePath, input, "headless", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } defer page.Close() - request.options.Output.Request(request.options.TemplateID, input, "headless", nil) + request.options.Output.Request(request.options.TemplatePath, input, "headless", nil) request.options.Progress.IncrementRequests() gologger.Verbose().Msgf("Sent Headless request to %s", input) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 3451e5f3..3dc5d3be 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -356,7 +356,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate _, _ = io.CopyN(ioutil.Discard, resp.Body, drainReqSize) resp.Body.Close() } - request.options.Output.Request(request.options.TemplateID, formedURL, "http", err) + request.options.Output.Request(request.options.TemplatePath, formedURL, "http", err) request.options.Progress.IncrementErrorsBy(1) // If we have interactsh markers and request times out, still send @@ -394,7 +394,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", request.options.TemplateID, formedURL) - request.options.Output.Request(request.options.TemplateID, formedURL, "http", err) + request.options.Output.Request(request.options.TemplatePath, formedURL, "http", err) duration := time.Since(timeStart) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 66da31e9..6c2fe661 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -36,7 +36,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review address, err = getAddress(input) } if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "network", err) + request.options.Output.Request(request.options.TemplatePath, input, "network", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not get address from url") } @@ -65,7 +65,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review func (request *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { if !strings.Contains(actualAddress, ":") { err := errors.New("no port provided in network protocol request") - request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, "network", err) request.options.Progress.IncrementFailedRequestsBy(1) return err } @@ -113,7 +113,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input conn, err = request.dialer.Dial(context.Background(), "tcp", actualAddress) } if err != nil { - request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, "network", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server request") } @@ -143,7 +143,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input data = []byte(input.Data) } if err != nil { - request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, "network", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } @@ -151,7 +151,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input finalData, dataErr := expressions.EvaluateByte(data, payloads) if dataErr != nil { - request.options.Output.Request(request.options.TemplateID, address, "network", dataErr) + request.options.Output.Request(request.options.TemplatePath, address, "network", dataErr) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } @@ -162,7 +162,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input return nil } if _, err := conn.Write(finalData); err != nil { - request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, "network", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } @@ -194,7 +194,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input gologger.Print().Msgf("%s\nHex: %s", requestOutput, hex.EncodeToString([]byte(requestOutput))) } - request.options.Output.Request(request.options.TemplateID, actualAddress, "network", err) + request.options.Output.Request(request.options.TemplatePath, actualAddress, "network", err) gologger.Verbose().Msgf("Sent TCP request to %s", actualAddress) bufferSize := 1024 @@ -225,7 +225,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input buf := make([]byte, bufferSize) nBuf, err := conn.Read(buf) if err != nil && !os.IsTimeout(err) { - request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, "network", err) closeTimer(readInterval) return errors.Wrap(err, "could not read from server") } @@ -238,7 +238,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input final = make([]byte, bufferSize) n, err = conn.Read(final) if err != nil && err != io.EOF { - request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, "network", err) return errors.Wrap(err, "could not read from server") } responseBuilder.Write(final[:n]) diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 1eb4bfc5..c9dbc1a8 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -91,7 +91,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review }) wg.Wait() if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "file", err) + request.options.Output.Request(request.options.TemplatePath, input, "file", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } From 1eb0378952b576db290fa15fc87fc6904ace13e5 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 30 Oct 2021 13:46:07 +0300 Subject: [PATCH 063/146] Unwrap errors in json log output --- v2/pkg/output/output.go | 5 +++-- v2/pkg/utils/utils.go | 12 ++++++++++++ v2/pkg/utils/utils_test.go | 21 +++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 v2/pkg/utils/utils_test.go diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 57c841a8..0b0cbf02 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/utils" ) // Writer is an interface which writes output to somewhere for nuclei events. @@ -192,8 +193,8 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques Input: input, Type: requestType, } - if requestErr != nil { - request.Error = requestErr.Error() + if unwrappedErr := utils.UnwrapError(requestErr); unwrappedErr != nil { + request.Error = unwrappedErr.Error() } else { request.Error = "none" } diff --git a/v2/pkg/utils/utils.go b/v2/pkg/utils/utils.go index 8d6fc0cf..ee924761 100644 --- a/v2/pkg/utils/utils.go +++ b/v2/pkg/utils/utils.go @@ -1,6 +1,7 @@ package utils import ( + "errors" "strings" ) @@ -11,3 +12,14 @@ func IsBlank(value string) bool { func IsNotBlank(value string) bool { return !IsBlank(value) } + +func UnwrapError(err error) error { + for { // get the last wrapped error + unwrapped := errors.Unwrap(err) + if unwrapped == nil { + break + } + err = unwrapped + } + return err +} diff --git a/v2/pkg/utils/utils_test.go b/v2/pkg/utils/utils_test.go new file mode 100644 index 00000000..ede9d530 --- /dev/null +++ b/v2/pkg/utils/utils_test.go @@ -0,0 +1,21 @@ +package utils + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUnwrapError(t *testing.T) { + require.Equal(t, nil, UnwrapError(nil)) + + errOne := fmt.Errorf("error one") + require.Equal(t, errOne, UnwrapError(errOne)) + + errTwo := fmt.Errorf("error with error: %w", errOne) + require.Equal(t, errOne, UnwrapError(errTwo)) + + errThree := fmt.Errorf("error with error: %w", errTwo) + require.Equal(t, errOne, UnwrapError(errThree)) +} From cd2db280bfd5469f5984d1d3f1bb96828cbdb6c8 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 30 Oct 2021 14:13:35 +0300 Subject: [PATCH 064/146] Move output mutex to fileWriter --- v2/pkg/output/file_output_writer.go | 6 ++++++ v2/pkg/output/output.go | 11 ----------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/v2/pkg/output/file_output_writer.go b/v2/pkg/output/file_output_writer.go index 94765ae2..84df0ab5 100644 --- a/v2/pkg/output/file_output_writer.go +++ b/v2/pkg/output/file_output_writer.go @@ -2,11 +2,13 @@ package output import ( "os" + "sync" ) // fileWriter is a concurrent file based output writer. type fileWriter struct { file *os.File + mu sync.Mutex } // NewFileOutputWriter creates a new buffered writer for a file @@ -20,6 +22,8 @@ func newFileOutputWriter(file string) (*fileWriter, error) { // WriteString writes an output to the underlying file func (w *fileWriter) Write(data []byte) error { + w.mu.Lock() + defer w.mu.Unlock() if _, err := w.file.Write(data); err != nil { return err } @@ -29,6 +33,8 @@ func (w *fileWriter) Write(data []byte) error { // Close closes the underlying writer flushing everything to disk func (w *fileWriter) Close() error { + w.mu.Lock() + defer w.mu.Unlock() //nolint:errcheck // we don't care whether sync failed or succeeded. w.file.Sync() return w.file.Close() diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 0b0cbf02..7189beb4 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -3,7 +3,6 @@ package output import ( "os" "regexp" - "sync" "time" "github.com/pkg/errors" @@ -39,11 +38,8 @@ type StandardWriter struct { noMetadata bool aurora aurora.Aurora outputFile *fileWriter - outputMutex *sync.Mutex traceFile *fileWriter - traceMutex *sync.Mutex errorFile *fileWriter - errorMutex *sync.Mutex severityColors func(severity.Severity) string } @@ -134,11 +130,8 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, noTimestamp: noTimestamp, aurora: auroraColorizer, outputFile: outputFile, - outputMutex: &sync.Mutex{}, traceFile: traceOutput, - traceMutex: &sync.Mutex{}, errorFile: errorOutput, - errorMutex: &sync.Mutex{}, severityColors: colorizer.New(auroraColorizer), } return writer, nil @@ -205,15 +198,11 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques } if w.traceFile != nil { - w.traceMutex.Lock() _ = w.traceFile.Write(data) - w.traceMutex.Unlock() } if requestErr != nil && w.errorFile != nil { - w.errorMutex.Lock() _ = w.errorFile.Write(data) - w.errorMutex.Unlock() } } From b8ebbc27f5a11f6d5edd74a5415bc0041893850e Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 30 Oct 2021 14:19:11 +0300 Subject: [PATCH 065/146] Use io.WriteCloser in output writer --- v2/pkg/output/file_output_writer.go | 10 ++++++---- v2/pkg/output/output.go | 19 ++++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/v2/pkg/output/file_output_writer.go b/v2/pkg/output/file_output_writer.go index 84df0ab5..8c1ab0eb 100644 --- a/v2/pkg/output/file_output_writer.go +++ b/v2/pkg/output/file_output_writer.go @@ -21,14 +21,16 @@ func newFileOutputWriter(file string) (*fileWriter, error) { } // WriteString writes an output to the underlying file -func (w *fileWriter) Write(data []byte) error { +func (w *fileWriter) Write(data []byte) (int, error) { w.mu.Lock() defer w.mu.Unlock() if _, err := w.file.Write(data); err != nil { - return err + return 0, err } - _, err := w.file.Write([]byte("\n")) - return err + if _, err := w.file.Write([]byte("\n")); err != nil { + return 0, err + } + return len(data) + 1, nil } // Close closes the underlying writer flushing everything to disk diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 7189beb4..991c9923 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -1,6 +1,7 @@ package output import ( + "io" "os" "regexp" "time" @@ -37,9 +38,9 @@ type StandardWriter struct { noTimestamp bool noMetadata bool aurora aurora.Aurora - outputFile *fileWriter - traceFile *fileWriter - errorFile *fileWriter + outputFile io.WriteCloser + traceFile io.WriteCloser + errorFile io.WriteCloser severityColors func(severity.Severity) string } @@ -99,7 +100,7 @@ type ResultEvent struct { func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, file, traceFile string, errorFile string) (*StandardWriter, error) { auroraColorizer := aurora.NewAurora(colors) - var outputFile *fileWriter + var outputFile io.WriteCloser if file != "" { output, err := newFileOutputWriter(file) if err != nil { @@ -107,7 +108,7 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, } outputFile = output } - var traceOutput *fileWriter + var traceOutput io.WriteCloser if traceFile != "" { output, err := newFileOutputWriter(traceFile) if err != nil { @@ -115,7 +116,7 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, } traceOutput = output } - var errorOutput *fileWriter + var errorOutput io.WriteCloser if errorFile != "" { output, err := newFileOutputWriter(errorFile) if err != nil { @@ -161,7 +162,7 @@ func (w *StandardWriter) Write(event *ResultEvent) error { if !w.json { data = decolorizerRegex.ReplaceAll(data, []byte("")) } - if writeErr := w.outputFile.Write(data); writeErr != nil { + if _, writeErr := w.outputFile.Write(data); writeErr != nil { return errors.Wrap(err, "could not write to output") } } @@ -198,11 +199,11 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques } if w.traceFile != nil { - _ = w.traceFile.Write(data) + _, _ = w.traceFile.Write(data) } if requestErr != nil && w.errorFile != nil { - _ = w.errorFile.Write(data) + _, _ = w.errorFile.Write(data) } } From bccc8e921b9aa640fd244f9d149ace0723e704ab Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 30 Oct 2021 15:04:16 +0300 Subject: [PATCH 066/146] Add test for output --- v2/pkg/output/output_test.go | 59 ++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 v2/pkg/output/output_test.go diff --git a/v2/pkg/output/output_test.go b/v2/pkg/output/output_test.go new file mode 100644 index 00000000..9f1d188b --- /dev/null +++ b/v2/pkg/output/output_test.go @@ -0,0 +1,59 @@ +package output + +import ( + "fmt" + "strings" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +func TestStandardWriterRequest(t *testing.T) { + t.Run("WithoutTraceAndError", func(t *testing.T) { + w, err := NewStandardWriter(false, false, false, false, false, "", "", "") + require.NoError(t, err) + require.NotPanics(t, func() { + w.Request("path", "input", "http", nil) + w.Close() + }) + }) + + t.Run("TraceAndErrorWithoutError", func(t *testing.T) { + traceWriter := &testWriteCloser{} + errorWriter := &testWriteCloser{} + + w, err := NewStandardWriter(false, false, false, false, false, "", "", "") + w.traceFile = traceWriter + w.errorFile = errorWriter + require.NoError(t, err) + w.Request("path", "input", "http", nil) + + require.Equal(t, `{"template":"path","input":"input","error":"none","type":"http"}`, traceWriter.String()) + require.Empty(t, errorWriter.String()) + }) + + t.Run("ErrorWithWrappedError", func(t *testing.T) { + errorWriter := &testWriteCloser{} + + w, err := NewStandardWriter(false, false, false, false, false, "", "", "") + w.errorFile = errorWriter + require.NoError(t, err) + w.Request( + "misconfiguration/tcpconfig.yaml", + "https://example.com/tcpconfig.html", + "http", + fmt.Errorf("GET https://example.com/tcpconfig.html/tcpconfig.html giving up after 2 attempts: %w", errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")), + ) + + require.Equal(t, `{"template":"misconfiguration/tcpconfig.yaml","input":"https://example.com/tcpconfig.html","error":"context deadline exceeded (Client.Timeout exceeded while awaiting headers)","type":"http"}`, errorWriter.String()) + }) +} + +type testWriteCloser struct { + strings.Builder +} + +func (w testWriteCloser) Close() error { + return nil +} From 38f2cf245e36e1139aa0b0683479c767828494f9 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Sat, 30 Oct 2021 16:41:10 +0300 Subject: [PATCH 067/146] feat: In case of binary data, show a hexadecimal view as well #1080 * added Compact hex view to the output if the -vv flag is provided --- v2/pkg/protocols/network/request.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 12e34e8d..c7c31e07 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -189,9 +189,12 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input request.options.Progress.IncrementRequests() if request.options.Options.Debug || request.options.Options.DebugRequests { - requestOutput := reqBuilder.String() gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n", request.options.TemplateID, actualAddress) - gologger.Print().Msgf("%s", hex.Dump([]byte(requestOutput))) + requestBytes := []byte(reqBuilder.String()) + gologger.Print().Msgf("%s", hex.Dump(requestBytes)) + if request.options.Options.VerboseVerbose { + gologger.Print().Msgf("\nCompact HEX view:\n%s", hex.EncodeToString(requestBytes)) + } } request.options.Output.Request(request.options.TemplateID, actualAddress, "network", err) @@ -282,7 +285,18 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input func debug(event *output.InternalWrappedEvent, request *Request, response string, actualAddress string) { if request.options.Options.Debug || request.options.Options.DebugResponse { gologger.Debug().Msgf("[%s] Dumped Network response for %s\n", request.options.TemplateID, actualAddress) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, hex.Dump([]byte(response)), request.options.Options.NoColor, true)) + requestBytes := []byte(response) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, hex.Dump(requestBytes), request.options.Options.NoColor, true)) + if request.options.Options.VerboseVerbose { + var allMatches []string + for _, namedMatch := range event.OperatorsResult.Matches { + for _, matchElement := range namedMatch { + allMatches = append(allMatches, hex.EncodeToString([]byte(matchElement))) + } + } + event.OperatorsResult.Matches["compact"] = allMatches + gologger.Print().Msgf("\nCompact HEX view:\n%s", responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), request.options.Options.NoColor, false)) + } } } From e1c39f255cdc674fde32a8a3fa750e7e21d25e45 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 1 Nov 2021 09:46:34 +0000 Subject: [PATCH 068/146] Auto Generate Syntax Docs + JSONSchema [Mon Nov 1 09:46:34 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 40 +++++++++++++++++++++++++++++++ nuclei-jsonschema.json | 10 ++++++++ v2/pkg/templates/templates_doc.go | 22 +++++++++++++++-- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index a3fffe35..3d16bda7 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -1554,6 +1554,26 @@ Valid values:
+
+ +case-insensitive bool + +
+
+ +CaseInsensitive enables case-insensitive matches. Default is false. + + +Valid values: + + + - false + + - true +
+ +
+ @@ -1833,6 +1853,26 @@ in the next request for some protocols (like HTTP).
+
+ +case-insensitive bool + +
+
+ +CaseInsensitive enables case-insensitive extractions. Default is false. + + +Valid values: + + + - false + + - true +
+ +
+ diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 1db207d7..4f22c386 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -191,6 +191,11 @@ "type": "boolean", "title": "mark extracted value for internal variable use", "description": "Internal when set to true will allow using the value extracted in the next request for some protocols" + }, + "case-insensitive": { + "type": "boolean", + "title": "use case insensitive extract", + "description": "use case insensitive extract" } }, "additionalProperties": false, @@ -293,6 +298,11 @@ "type": "string", "title": "encoding for word field", "description": "Optional encoding for the word fields" + }, + "case-insensitive": { + "type": "boolean", + "title": "use case insensitive match", + "description": "use case insensitive match" } }, "additionalProperties": false, diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index d6042dfb..ca416b3b 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -473,7 +473,7 @@ func init() { FieldName: "matchers", }, } - MATCHERSMatcherDoc.Fields = make([]encoder.Doc, 12) + MATCHERSMatcherDoc.Fields = make([]encoder.Doc, 13) MATCHERSMatcherDoc.Fields[0].Name = "type" MATCHERSMatcherDoc.Fields[0].Type = "string" MATCHERSMatcherDoc.Fields[0].Note = "" @@ -575,6 +575,15 @@ func init() { MATCHERSMatcherDoc.Fields[11].Values = []string{ "hex", } + MATCHERSMatcherDoc.Fields[12].Name = "case-insensitive" + MATCHERSMatcherDoc.Fields[12].Type = "bool" + MATCHERSMatcherDoc.Fields[12].Note = "" + MATCHERSMatcherDoc.Fields[12].Description = "CaseInsensitive enables case-insensitive matches. Default is false." + MATCHERSMatcherDoc.Fields[12].Comments[encoder.LineComment] = "CaseInsensitive enables case-insensitive matches. Default is false." + MATCHERSMatcherDoc.Fields[12].Values = []string{ + "false", + "true", + } EXTRACTORSExtractorDoc.Type = "extractors.Extractor" EXTRACTORSExtractorDoc.Comments[encoder.LineComment] = " Extractor is used to extract part of response using a regex." @@ -601,7 +610,7 @@ func init() { FieldName: "extractors", }, } - EXTRACTORSExtractorDoc.Fields = make([]encoder.Doc, 10) + EXTRACTORSExtractorDoc.Fields = make([]encoder.Doc, 11) EXTRACTORSExtractorDoc.Fields[0].Name = "name" EXTRACTORSExtractorDoc.Fields[0].Type = "string" EXTRACTORSExtractorDoc.Fields[0].Note = "" @@ -678,6 +687,15 @@ func init() { EXTRACTORSExtractorDoc.Fields[9].Note = "" EXTRACTORSExtractorDoc.Fields[9].Description = "Internal, when set to true will allow using the value extracted\nin the next request for some protocols (like HTTP)." EXTRACTORSExtractorDoc.Fields[9].Comments[encoder.LineComment] = "Internal, when set to true will allow using the value extracted" + EXTRACTORSExtractorDoc.Fields[10].Name = "case-insensitive" + EXTRACTORSExtractorDoc.Fields[10].Type = "bool" + EXTRACTORSExtractorDoc.Fields[10].Note = "" + EXTRACTORSExtractorDoc.Fields[10].Description = "CaseInsensitive enables case-insensitive extractions. Default is false." + EXTRACTORSExtractorDoc.Fields[10].Comments[encoder.LineComment] = "CaseInsensitive enables case-insensitive extractions. Default is false." + EXTRACTORSExtractorDoc.Fields[10].Values = []string{ + "false", + "true", + } DNSRequestDoc.Type = "dns.Request" DNSRequestDoc.Comments[encoder.LineComment] = " Request contains a DNS protocol request to be made from a template" From bb05be7b952e595662db7415d6a9124fdcc8a7ea Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 1 Nov 2021 15:47:20 +0530 Subject: [PATCH 069/146] Added integration tests for websocket + misc fixes --- integration_tests/websocket/basic.yaml | 16 ++++ integration_tests/websocket/cswsh.yaml | 16 ++++ integration_tests/websocket/no-cswsh.yaml | 16 ++++ v2/cmd/integration-test/integration-test.go | 9 +- v2/cmd/integration-test/websocket.go | 90 +++++++++++++++++++ v2/go.mod | 1 + v2/go.sum | 2 + v2/internal/testutils/integration.go | 24 +++++ .../protocols/others/websocket/websocket.go | 16 ++-- v2/pkg/templates/templates.go | 6 ++ 10 files changed, 186 insertions(+), 10 deletions(-) create mode 100644 integration_tests/websocket/basic.yaml create mode 100644 integration_tests/websocket/cswsh.yaml create mode 100644 integration_tests/websocket/no-cswsh.yaml create mode 100644 v2/cmd/integration-test/websocket.go diff --git a/integration_tests/websocket/basic.yaml b/integration_tests/websocket/basic.yaml new file mode 100644 index 00000000..c0937830 --- /dev/null +++ b/integration_tests/websocket/basic.yaml @@ -0,0 +1,16 @@ +id: basic-request + +info: + name: Basic Request + author: pdteam + severity: info + +websocket: + - address: '{{Scheme}}://{{Hostname}}' + inputs: + - data: hello + matchers: + - type: word + words: + - world + part: response \ No newline at end of file diff --git a/integration_tests/websocket/cswsh.yaml b/integration_tests/websocket/cswsh.yaml new file mode 100644 index 00000000..80f75d7a --- /dev/null +++ b/integration_tests/websocket/cswsh.yaml @@ -0,0 +1,16 @@ +id: basic-cswsh-request + +info: + name: Basic cswsh Request + author: pdteam + severity: info + +websocket: + - address: '{{Scheme}}://{{Hostname}}' + headers: + Origin: 'http://evil.com' + matchers: + - type: word + words: + - true + part: success \ No newline at end of file diff --git a/integration_tests/websocket/no-cswsh.yaml b/integration_tests/websocket/no-cswsh.yaml new file mode 100644 index 00000000..6833d804 --- /dev/null +++ b/integration_tests/websocket/no-cswsh.yaml @@ -0,0 +1,16 @@ +id: basic-nocswsh-request + +info: + name: Basic Non-Vulnerable cswsh Request + author: pdteam + severity: info + +websocket: + - address: '{{Scheme}}://{{Hostname}}' + headers: + Origin: 'http://evil.com' + matchers: + - type: word + words: + - true + part: success \ No newline at end of file diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index baa743e8..41c035f6 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -22,10 +22,11 @@ func main() { failed := aurora.Red("[✘]").String() protocolTests := map[string]map[string]testutils.TestCase{ - "http": httpTestcases, - "network": networkTestcases, - "dns": dnsTestCases, - "workflow": workflowTestcases, + "http": httpTestcases, + "network": networkTestcases, + "dns": dnsTestCases, + "workflow": workflowTestcases, + "websocket": websocketTestCases, } for proto, tests := range protocolTests { if protocol == "" || protocol == proto { diff --git a/v2/cmd/integration-test/websocket.go b/v2/cmd/integration-test/websocket.go new file mode 100644 index 00000000..25f249af --- /dev/null +++ b/v2/cmd/integration-test/websocket.go @@ -0,0 +1,90 @@ +package main + +import ( + "net" + "strings" + + "github.com/gobwas/ws/wsutil" + "github.com/projectdiscovery/nuclei/v2/internal/testutils" +) + +var websocketTestCases = map[string]testutils.TestCase{ + "websocket/basic.yaml": &websocketBasic{}, + "websocket/cswsh.yaml": &websocketCswsh{}, + "websocket/no-cswsh.yaml": &websocketNoCswsh{}, +} + +type websocketBasic struct{} + +// Execute executes a test case and returns an error if occurred +func (h *websocketBasic) Execute(filePath string) error { + connHandler := func(conn net.Conn) { + for { + msg, op, _ := wsutil.ReadClientData(conn) + if string(msg) != string("hello") { + return + } + _ = wsutil.WriteServerMessage(conn, op, []byte("world")) + } + } + originValidate := func(origin string) bool { + return true + } + ts := testutils.NewWebsocketServer(connHandler, originValidate) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + +type websocketCswsh struct{} + +// Execute executes a test case and returns an error if occurred +func (h *websocketCswsh) Execute(filePath string) error { + connHandler := func(conn net.Conn) { + + } + originValidate := func(origin string) bool { + return true + } + ts := testutils.NewWebsocketServer(connHandler, originValidate) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + +type websocketNoCswsh struct{} + +// Execute executes a test case and returns an error if occurred +func (h *websocketNoCswsh) Execute(filePath string) error { + connHandler := func(conn net.Conn) { + + } + originValidate := func(origin string) bool { + return origin == "https://google.com" + } + ts := testutils.NewWebsocketServer(connHandler, originValidate) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) + if err != nil { + return err + } + if len(results) != 0 { + return errIncorrectResultsCount(results) + } + return nil +} diff --git a/v2/go.mod b/v2/go.mod index 9d3d5017..3596218c 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -100,6 +100,7 @@ require ( github.com/mattn/go-runewidth v0.0.13 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect diff --git a/v2/go.sum b/v2/go.sum index d4a21e8b..74aba21b 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -556,6 +556,8 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= +github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index 8c5491d2..13a3d8c6 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -4,10 +4,14 @@ import ( "errors" "fmt" "net" + "net/http" + "net/http/httptest" "os" "os/exec" "regexp" "strings" + + "github.com/gobwas/ws" ) // RunNucleiTemplateAndGetResults returns a list of results for a template @@ -113,3 +117,23 @@ func NewTCPServer(handler func(conn net.Conn), port ...int) *TCPServer { func (s *TCPServer) Close() { s.listener.Close() } + +// NewWebsocketServer creates a new websocket server from a handler +func NewWebsocketServer(handler func(conn net.Conn), originValidate func(origin string) bool, port ...int) *httptest.Server { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if value := r.Header.Get("Origin"); value != "" && !originValidate(value) { + w.WriteHeader(http.StatusBadRequest) + return + } + conn, _, _, err := ws.UpgradeHTTP(r, w) + if err != nil { + return + } + go func() { + defer conn.Close() + + handler(conn) + }() + })) + return ts +} diff --git a/v2/pkg/protocols/others/websocket/websocket.go b/v2/pkg/protocols/others/websocket/websocket.go index 7c855854..ec73e827 100644 --- a/v2/pkg/protocols/others/websocket/websocket.go +++ b/v2/pkg/protocols/others/websocket/websocket.go @@ -99,7 +99,7 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if len(r.Payloads) > 0 { attackType := r.AttackType if attackType == "" { - attackType = "sniper" + attackType = "batteringram" } r.attackType = generators.StringToType[attackType] @@ -183,10 +183,10 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu if err != nil { return errors.Wrap(err, "could not parse input url") } - payloadValues["Address"] = parsed.Host + payloadValues["Hostname"] = parsed.Host + payloadValues["Host"] = parsed.Hostname() payloadValues["Scheme"] = parsed.Scheme payloadValues["Path"] = parsed.Path - payloadValues["hostname"] = parsed.Hostname() for key, value := range r.Headers { finalData, dataErr := expressions.EvaluateByte([]byte(value), payloadValues) @@ -210,10 +210,14 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu r.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } + addressToDial := string(finalAddress) + if parsed.Path != "" && parsed.Path != "/" { + addressToDial = addressToDial + parsed.Path + } - conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), string(finalAddress)) + conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial) if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "ssl", err) + r.options.Output.Request(r.options.TemplateID, input, "websocket", err) r.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server") } @@ -296,7 +300,7 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu data["ip"] = r.dialer.GetDialedIP(hostname) event := eventcreator.CreateEventWithAdditionalOptions(r, data, r.options.Options.Debug || r.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { - internalWrappedEvent.OperatorsResult.PayloadValues = payloadValues + internalWrappedEvent.OperatorsResult.PayloadValues = dynamicValues }) if r.options.Options.Debug || r.options.Options.DebugResponse { responseOutput := responseBuilder.String() diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index f71fa1df..79e6d766 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -90,6 +90,8 @@ var TemplateTypes = []string{ "headless", "network", "workflow", + "ssl", + "websocket", } // Type returns the type of the template @@ -107,6 +109,10 @@ func (t *Template) Type() string { return "network" case len(t.Workflows) > 0: return "workflow" + case len(t.RequestsSSL) > 0: + return "ssl" + case len(t.RequestsWebsocket) > 0: + return "websocket" default: return "" } From a274cc572254d8d0404a21726271b90b0efe7b62 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 1 Nov 2021 15:51:56 +0530 Subject: [PATCH 070/146] Misc integration test --- integration_tests/websocket/path.yaml | 16 ++++++++++++++ v2/cmd/integration-test/websocket.go | 30 ++++++++++++++++++++++++--- v2/internal/testutils/integration.go | 19 ++++++++++++++--- 3 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 integration_tests/websocket/path.yaml diff --git a/integration_tests/websocket/path.yaml b/integration_tests/websocket/path.yaml new file mode 100644 index 00000000..d3607b30 --- /dev/null +++ b/integration_tests/websocket/path.yaml @@ -0,0 +1,16 @@ +id: basic-request-path + +info: + name: Basic Request Path + author: pdteam + severity: info + +websocket: + - address: '{{Scheme}}://{{Hostname}}' + inputs: + - data: hello + matchers: + - type: word + words: + - world + part: response \ No newline at end of file diff --git a/v2/cmd/integration-test/websocket.go b/v2/cmd/integration-test/websocket.go index 25f249af..be20ba77 100644 --- a/v2/cmd/integration-test/websocket.go +++ b/v2/cmd/integration-test/websocket.go @@ -12,6 +12,7 @@ var websocketTestCases = map[string]testutils.TestCase{ "websocket/basic.yaml": &websocketBasic{}, "websocket/cswsh.yaml": &websocketCswsh{}, "websocket/no-cswsh.yaml": &websocketNoCswsh{}, + "websocket/path.yaml": &websocketWithPath{}, } type websocketBasic struct{} @@ -30,7 +31,7 @@ func (h *websocketBasic) Execute(filePath string) error { originValidate := func(origin string) bool { return true } - ts := testutils.NewWebsocketServer(connHandler, originValidate) + ts := testutils.NewWebsocketServer("", connHandler, originValidate) defer ts.Close() results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) @@ -53,7 +54,7 @@ func (h *websocketCswsh) Execute(filePath string) error { originValidate := func(origin string) bool { return true } - ts := testutils.NewWebsocketServer(connHandler, originValidate) + ts := testutils.NewWebsocketServer("", connHandler, originValidate) defer ts.Close() results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) @@ -76,7 +77,30 @@ func (h *websocketNoCswsh) Execute(filePath string) error { originValidate := func(origin string) bool { return origin == "https://google.com" } - ts := testutils.NewWebsocketServer(connHandler, originValidate) + ts := testutils.NewWebsocketServer("", connHandler, originValidate) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) + if err != nil { + return err + } + if len(results) != 0 { + return errIncorrectResultsCount(results) + } + return nil +} + +type websocketWithPath struct{} + +// Execute executes a test case and returns an error if occurred +func (h *websocketWithPath) Execute(filePath string) error { + connHandler := func(conn net.Conn) { + + } + originValidate := func(origin string) bool { + return origin == "https://google.com" + } + ts := testutils.NewWebsocketServer("/test", connHandler, originValidate) defer ts.Close() results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index 13a3d8c6..4c3f84e3 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/gobwas/ws" + "github.com/julienschmidt/httprouter" ) // RunNucleiTemplateAndGetResults returns a list of results for a template @@ -119,8 +120,8 @@ func (s *TCPServer) Close() { } // NewWebsocketServer creates a new websocket server from a handler -func NewWebsocketServer(handler func(conn net.Conn), originValidate func(origin string) bool, port ...int) *httptest.Server { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +func NewWebsocketServer(path string, handler func(conn net.Conn), originValidate func(origin string) bool, port ...int) *httptest.Server { + handlerFunc := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if value := r.Header.Get("Origin"); value != "" && !originValidate(value) { w.WriteHeader(http.StatusBadRequest) return @@ -134,6 +135,18 @@ func NewWebsocketServer(handler func(conn net.Conn), originValidate func(origin handler(conn) }() - })) + }) + + var router *httprouter.Router + if path != "" { + router = httprouter.New() + router.HandlerFunc("*", "/test", handlerFunc) + } + var ts *httptest.Server + if router != nil { + ts = httptest.NewServer(router) + } else { + ts = httptest.NewServer(handlerFunc) + } return ts } From f8c6f0ba647aa37c3634e259751c09e405098dc7 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 1 Nov 2021 18:02:45 +0530 Subject: [PATCH 071/146] Added debug for SSL --- v2/pkg/protocols/others/ssl/ssl.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/others/ssl/ssl.go b/v2/pkg/protocols/others/ssl/ssl.go index ec6a32a1..f8f34b00 100644 --- a/v2/pkg/protocols/others/ssl/ssl.go +++ b/v2/pkg/protocols/others/ssl/ssl.go @@ -8,7 +8,9 @@ import ( "strings" "time" + jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" + "github.com/projectdiscovery/cryptoutil" "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/operators" @@ -17,6 +19,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -71,6 +74,7 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu hostname, _, _ := net.SplitHostPort(address) config := &tls.Config{InsecureSkipVerify: true, ServerName: hostname} + conn, err := r.dialer.DialTLSWithConfig(context.Background(), "tcp", address, config) if err != nil { r.options.Output.Request(r.options.TemplateID, input, "ssl", err) @@ -87,16 +91,33 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu r.options.Output.Request(r.options.TemplateID, address, "ssl", err) gologger.Verbose().Msgf("Sent SSL request to %s", address) - if len(connTLS.ConnectionState().PeerCertificates) == 0 { + if r.options.Options.Debug || r.options.Options.DebugRequests { + gologger.Info().Str("address", input).Msgf("[%s] Dumped SSL request for %s", r.options.TemplateID, input) + } + + state := connTLS.ConnectionState() + if len(state.PeerCertificates) == 0 { return nil } + + tlsData := cryptoutil.TLSGrab(&state) + jsonData, _ := jsoniter.Marshal(tlsData) + jsonDataString := string(jsonData) + data := make(map[string]interface{}) cert := connTLS.ConnectionState().PeerCertificates[0] + + data["response"] = jsonDataString data["host"] = input data["not_after"] = float64(cert.NotAfter.Unix()) data["ip"] = r.dialer.GetDialedIP(hostname) event := eventcreator.CreateEvent(r, data, r.options.Options.Debug || r.options.Options.DebugResponse) + if r.options.Options.Debug || r.options.Options.DebugResponse { + responseOutput := jsonDataString + gologger.Debug().Msgf("[%s] Dumped SSL response for %s", r.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, r.options.Options.NoColor)) + } callback(event) return nil } From b4775085215510402b2be6d9ec77b0f819146620 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Mon, 1 Nov 2021 20:44:55 +0200 Subject: [PATCH 072/146] feat: In case of binary data, show a hexadecimal view as well #1080 * the ASCII column in the hex dump represents non-printable ASCII characters with a "." character, so in order to enable proper highlighting, those characters has to be replaced in the generated regex as well --- .../helpers/responsehighlighter/hexdump.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go b/v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go index 10c2cfc5..decc985c 100644 --- a/v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go @@ -77,7 +77,13 @@ func highlightHexSection(hexDump HighlightableHexDump, snippetToColor string) Hi func highlightAsciiSection(hexDump HighlightableHexDump, snippetToColor string) HighlightableHexDump { var snippetCharactersMatchPattern string for _, v := range snippetToColor { - snippetCharactersMatchPattern += fmt.Sprintf(`(%s\n*)`, regexp.QuoteMeta(string(v))) + var value string + if IsASCIIPrintable(v) { + value = regexp.QuoteMeta(string(v)) + } else { + value = "." + } + snippetCharactersMatchPattern += fmt.Sprintf(`(%s\n*)`, value) } hexDump.ascii = highlight(hexDump.ascii, snippetCharactersMatchPattern, func(v string) string { @@ -105,6 +111,10 @@ func highlight(values []string, snippetCharactersMatchPattern string, replaceToF return strings.Split(rows, "\n") } +func HasBinaryContent(input string) bool { + return !IsASCII(input) +} + // IsASCII tests whether a string consists only of ASCII characters or not func IsASCII(input string) bool { for i := 0; i < len(input); i++ { @@ -114,3 +124,7 @@ func IsASCII(input string) bool { } return true } + +func IsASCIIPrintable(input rune) bool { + return input > 32 && input < unicode.MaxASCII +} From 8f6280dc9dcd1bebebc09ec66cb3bb79c0ef0887 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Mon, 1 Nov 2021 20:45:54 +0200 Subject: [PATCH 073/146] refactor: In case of binary data, show a hexadecimal view as well #1080 * small enhancements with regards to dumping responses --- v2/pkg/protocols/dns/request.go | 14 ++++----- v2/pkg/protocols/file/request.go | 14 ++++----- v2/pkg/protocols/headless/request.go | 11 +++---- v2/pkg/protocols/http/request.go | 43 ++++++++++++++++------------ v2/pkg/protocols/network/request.go | 41 +++++++++++++++----------- 5 files changed, 70 insertions(+), 53 deletions(-) diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index d063825f..08422b8e 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -65,22 +65,22 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - debug(event, request, domain, response.String()) + dumpResponse(event, request.options, response.String(), domain) callback(event) return nil } -func debug(event *output.InternalWrappedEvent, request *Request, domain string, response string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped DNS response for %s\n", request.options.TemplateID, domain) - +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response string, domain string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { hexDump := false - if !responsehighlighter.IsASCII(response) { + if responsehighlighter.HasBinaryContent(response) { hexDump = true response = hex.Dump([]byte(response)) } - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump)) + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, response, cliOptions.NoColor, hexDump) + gologger.Debug().Msgf("[%s] Dumped DNS response for %s\n\n%s", requestOptions.TemplateID, domain, highlightedResponse) } } diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index c7c3daee..0326de37 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -2,7 +2,6 @@ package file import ( "encoding/hex" - "fmt" "io/ioutil" "os" @@ -61,7 +60,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - debug(event, request, filePath, fileContent) + dumpResponse(event, request.options, fileContent, filePath) callback(event) }(data) @@ -76,14 +75,15 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review return nil } -func debug(event *output.InternalWrappedEvent, request *Request, filePath string, fileContent string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, fileContent string, filePath string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { hexDump := false - if !responsehighlighter.IsASCII(fileContent) { + if responsehighlighter.HasBinaryContent(fileContent) { hexDump = true fileContent = hex.Dump([]byte(fileContent)) } - logHeader := fmt.Sprintf("[%s] Dumped file request for %s\n", request.options.TemplateID, filePath) - gologger.Debug().Msgf("%s\n%s", logHeader, responsehighlighter.Highlight(event.OperatorsResult, fileContent, request.options.Options.NoColor, hexDump)) + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, fileContent, cliOptions.NoColor, hexDump) + gologger.Debug().Msgf("[%s] Dumped file request for %s\n\n%s", requestOptions.TemplateID, filePath, highlightedResponse) } } diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 1609a595..f4c0618b 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -67,15 +67,16 @@ func (request *Request) ExecuteWithResults(inputURL string, metadata, previous o event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - debug(event, request, responseBody, inputURL) + dumpResponse(event, request.options, responseBody, inputURL) callback(event) return nil } -func debug(event *output.InternalWrappedEvent, request *Request, responseBody string, input string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n", request.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor, false)) +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, responseBody string, input string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, responseBody, cliOptions.NoColor, false) + gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n\n%s", requestOptions.TemplateID, input, highlightedResponse) } } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index b11340bf..28110d38 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -511,7 +511,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta }) - debug(request, formedURL, redirectedResponse, responseContentType, event) + dumpResponse(event, request.options, redirectedResponse, formedURL, responseContentType) callback(event) return nil @@ -534,26 +534,33 @@ func (request *Request) setCustomHeaders(req *generatedRequest) { const CRLF = "\r\n" -func debug(request *Request, formedURL string, redirectedResponse []byte, responseContentType string, event *output.InternalWrappedEvent) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - hexDump := false +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, redirectedResponse []byte, formedURL string, responseContentType string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { response := string(redirectedResponse) - var headers string - if responseContentType == "" || responseContentType == "application/octet-stream" || (responseContentType == "application/x-www-form-urlencoded" && responsehighlighter.IsASCII(response)) { - hexDump = true - responseLines := strings.Split(response, CRLF) - for i, value := range responseLines { - headers += value + CRLF - if value == "" { - response = hex.Dump([]byte(strings.Join(responseLines[i+1:], ""))) - break - } - } + var highlightedResult string + if responseContentType == "application/octet-stream" || ((responseContentType == "" || responseContentType == "application/x-www-form-urlencoded") && responsehighlighter.HasBinaryContent(response)) { + highlightedResult = createResponseHexDump(event, response, cliOptions.NoColor) + } else { + highlightedResult = responsehighlighter.Highlight(event.OperatorsResult, response, cliOptions.NoColor, false) } - logMessageHeader := fmt.Sprintf("[%s] Dumped HTTP response for %s\n", request.options.TemplateID, formedURL) - gologger.Debug().Msgf("%s\n%s", logMessageHeader, responsehighlighter.Highlight(event.OperatorsResult, headers, request.options.Options.NoColor, false)) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump)) + gologger.Debug().Msgf("[%s] Dumped HTTP response for %s\n\n%s", requestOptions.TemplateID, formedURL, highlightedResult) + } +} + +func createResponseHexDump(event *output.InternalWrappedEvent, response string, noColor bool) string { + CRLFs := CRLF + CRLF + headerEndIndex := strings.Index(response, CRLFs) + len(CRLFs) + if headerEndIndex > 0 { + headers := response[0:headerEndIndex] + responseBodyHexDump := hex.Dump([]byte(response[headerEndIndex:])) + + highlightedHeaders := responsehighlighter.Highlight(event.OperatorsResult, headers, noColor, false) + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, responseBodyHexDump, noColor, true) + return fmt.Sprintf("%s\n%s", highlightedHeaders, highlightedResponse) + } else { + return responsehighlighter.Highlight(event.OperatorsResult, hex.Dump([]byte(response)), noColor, true) } } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 76135085..fa1e1b60 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" @@ -189,9 +190,8 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input request.options.Progress.IncrementRequests() if request.options.Options.Debug || request.options.Options.DebugRequests { - gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n", request.options.TemplateID, actualAddress) requestBytes := []byte(reqBuilder.String()) - gologger.Print().Msgf("%s", hex.Dump(requestBytes)) + gologger.Debug().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n%s", request.options.TemplateID, actualAddress, hex.Dump(requestBytes)) if request.options.Options.VerboseVerbose { gologger.Print().Msgf("\nCompact HEX view:\n%s", hex.EncodeToString(requestBytes)) } @@ -277,29 +277,38 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input }) } - debug(event, request, response, actualAddress) + dumpResponse(event, request.options, response, actualAddress) return nil } -func debug(event *output.InternalWrappedEvent, request *Request, response string, actualAddress string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Network response for %s\n", request.options.TemplateID, actualAddress) +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response string, actualAddress string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { requestBytes := []byte(response) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, hex.Dump(requestBytes), request.options.Options.NoColor, true)) - if request.options.Options.VerboseVerbose { - var allMatches []string - for _, namedMatch := range event.OperatorsResult.Matches { - for _, matchElement := range namedMatch { - allMatches = append(allMatches, hex.EncodeToString([]byte(matchElement))) - } - } - event.OperatorsResult.Matches["compact"] = allMatches - gologger.Print().Msgf("\nCompact HEX view:\n%s", responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), request.options.Options.NoColor, false)) + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, hex.Dump(requestBytes), cliOptions.NoColor, true) + gologger.Debug().Msgf("[%s] Dumped Network response for %s\n\n%s", requestOptions.TemplateID, actualAddress, highlightedResponse) + + if cliOptions.VerboseVerbose { + displayCompactHexView(event, response, cliOptions.NoColor) } } } +func displayCompactHexView(event *output.InternalWrappedEvent, response string, noColor bool) { + operatorsResult := event.OperatorsResult + if operatorsResult != nil { + var allMatches []string + for _, namedMatch := range operatorsResult.Matches { + for _, matchElement := range namedMatch { + allMatches = append(allMatches, hex.EncodeToString([]byte(matchElement))) + } + } + tempOperatorResult := &operators.Result{Matches: map[string][]string{"matchesInHex": allMatches}} + gologger.Print().Msgf("\nCompact HEX view:\n%s", responsehighlighter.Highlight(tempOperatorResult, hex.EncodeToString([]byte(response)), noColor, false)) + } +} + // getAddress returns the address of the host to make request to func getAddress(toTest string) (string, error) { if strings.Contains(toTest, "://") { From 191797380e5385b39fe5a7e4a8288e34dca5848a Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 2 Nov 2021 14:12:59 +0530 Subject: [PATCH 074/146] bug: Bug in URL parsing for unsafe templates #830 Fixed raw request path not correct with unsafe --- v2/pkg/protocols/http/raw/raw.go | 23 +++++++++++++++++++---- v2/pkg/protocols/http/raw/raw_test.go | 11 +++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 2b3d44af..18bebc39 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -2,10 +2,12 @@ package raw import ( "bufio" + "bytes" "fmt" "io" "io/ioutil" "net/url" + "path/filepath" "strings" "github.com/projectdiscovery/rawhttp/client" @@ -27,6 +29,12 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { rawRequest := &Request{ Headers: make(map[string]string), } + + parsedURL, err := url.Parse(baseURL) + if err != nil { + return nil, fmt.Errorf("could not parse request URL: %s", err) + } + if unsafe { rawRequest.UnsafeRawBytes = []byte(request) } @@ -40,6 +48,11 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { if len(parts) < 3 && !unsafe { return nil, fmt.Errorf("malformed request supplied") } + // Check if we have also a path from the passed base URL and if yes, + // append that to the unsafe request as well. + if parsedURL.Path != "" && strings.HasPrefix(parts[1], "/") && parts[1] != parsedURL.Path { + rawRequest.UnsafeRawBytes = fixUnsafeRequestPath(parsedURL, parts[1], rawRequest.UnsafeRawBytes) + } // Set the request Method rawRequest.Method = parts[0] @@ -98,10 +111,6 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { rawRequest.Path = parts[1] } - parsedURL, err := url.Parse(baseURL) - if err != nil { - return nil, fmt.Errorf("could not parse request URL: %s", err) - } hostURL := parsedURL.Host if strings.HasSuffix(parsedURL.Path, "/") && strings.HasPrefix(rawRequest.Path, "/") { parsedURL.Path = strings.TrimSuffix(parsedURL.Path, "/") @@ -131,3 +140,9 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { } return rawRequest, nil } + +func fixUnsafeRequestPath(baseURL *url.URL, requestPath string, request []byte) []byte { + fixedPath := filepath.Join(baseURL.Path, requestPath) + fixed := bytes.Replace(request, []byte(requestPath), []byte(fixedPath), 1) + return fixed +} diff --git a/v2/pkg/protocols/http/raw/raw_test.go b/v2/pkg/protocols/http/raw/raw_test.go index 540fb017..0e7cee9b 100644 --- a/v2/pkg/protocols/http/raw/raw_test.go +++ b/v2/pkg/protocols/http/raw/raw_test.go @@ -64,3 +64,14 @@ username=admin&password=login`, "https://test.com", false) require.Equal(t, "POST", request.Method, "Could not parse POST method request correctly") require.Equal(t, "username=admin&password=login", request.Data, "Could not parse request data correctly") } + +func TestParseUnsafeRequestWithPath(t *testing.T) { + request, err := Parse(`GET /manager/html HTTP/1.1 +Host: {{Hostname}} +Authorization: Basic {{base64('username:password')}} +User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0 +Accept-Language: en-US,en;q=0.9 +Connection: close`, "https://test.com/test/", true) + require.Nil(t, err, "could not parse unsafe request") + require.Contains(t, string(request.UnsafeRawBytes), "GET /test/manager/html", "Could not parse unsafe method request path correctly") +} From 87c3071ba7cecb6708003310ccfc687444678bd7 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 2 Nov 2021 14:59:05 +0530 Subject: [PATCH 075/146] Misc --- v2/go.mod | 4 ++++ v2/go.sum | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/v2/go.mod b/v2/go.mod index 3596218c..df29aa9d 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -113,12 +113,16 @@ require ( github.com/trivago/tgo v1.0.7 // indirect github.com/ulikunitz/xz v0.5.10 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect github.com/ysmood/goob v0.3.0 // indirect github.com/ysmood/gson v0.6.4 // indirect github.com/ysmood/leakless v0.7.0 // indirect github.com/zclconf/go-cty v1.8.4 // indirect + github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect + github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b // indirect go.etcd.io/bbolt v1.3.6 // indirect + golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 // indirect golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index 74aba21b..1ff2440b 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -505,6 +505,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= @@ -680,6 +681,7 @@ github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bs github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -754,6 +756,8 @@ github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95 h1:DyAZOw3JsVd6LJHqhl4MpKQdYQEmat0C6pPPwom39Ow= +github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= @@ -792,6 +796,11 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zclconf/go-cty v1.8.4 h1:pwhhz5P+Fjxse7S7UriBrMu6AUJSZM5pKqGem1PjGAs= github.com/zclconf/go-cty v1.8.4/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 h1:kKCF7VX/wTmdg2ZjEaqlq99Bjsoiz7vH6sFniF/vI4M= +github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= +github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= +github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b h1:iYQzlljG1dOXBtsJGyzFC/wBK5qUCWs1eLCr/UcJYPA= +github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b/go.mod h1:5nID//bFGkx3/+iHcFIFRHQ54EOPJ0iSj0IGKpMElvw= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= @@ -839,6 +848,8 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 h1:xYJJ3S178yv++9zXV/hnr29plCAGO9vAFG9dorqaFQc= +golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -918,6 +929,7 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= @@ -1019,6 +1031,7 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw= golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 2224880d891f1e9a1dff4819c501f5d5c95436b9 Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 2 Nov 2021 22:57:00 +0530 Subject: [PATCH 076/146] misc flag update --- v2/cmd/nuclei/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 08905537..7f43868f 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -54,10 +54,10 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "templates", "Templates", flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"), - flagSet.StringSliceVarP(&options.TemplateURLs, "template-urls", "tu", []string{}, "URL to a list of templates"), + flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "tu", []string{}, "URL containing list of templates to run"), flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"), flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "workflow or workflow directory paths to include in the scan"), - flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-urls", "wu", []string{}, "URL to a list of workflows to run"), + flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wu", []string{}, "URL containing list of workflows to run"), flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"), flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), ) From bdb415b0c7f513bcb9acd7c5989b524b02aa38e3 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 3 Nov 2021 02:23:48 +0530 Subject: [PATCH 077/146] Misc changes as per review --- v2/cmd/nuclei/main.go | 2 +- v2/pkg/catalog/loader/loader.go | 2 +- v2/pkg/core/engine.go | 10 ++++++---- v2/pkg/parsers/workflow_loader.go | 2 +- v2/pkg/types/types.go | 4 ++-- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 13602bfc..08cc082a 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -68,7 +68,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), - flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), + flagSet.NormalizedStringSliceVarP(&options.Authors, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), ) createGroup(flagSet, "output", "Output", diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 386898b1..9915cf49 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -54,7 +54,7 @@ func NewConfig(options *types.Options, catalog *catalog.Catalog, executerOpts pr Tags: options.Tags, ExcludeTags: options.ExcludeTags, IncludeTemplates: options.IncludeTemplates, - Authors: options.Author, + Authors: options.Authors, Severities: options.Severities, ExcludeSeverities: options.ExcludeSeverities, IncludeTags: options.IncludeTags, diff --git a/v2/pkg/core/engine.go b/v2/pkg/core/engine.go index 45554624..5d7cf946 100644 --- a/v2/pkg/core/engine.go +++ b/v2/pkg/core/engine.go @@ -5,7 +5,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" ) -// Engine is an engine for running Nuclei Templates/Workflows. +// Engine is an executer for running Nuclei Templates/Workflows. // // The engine contains multiple thread pools which allow using different // concurrency values per protocol executed. @@ -19,14 +19,16 @@ type Engine struct { executerOpts protocols.ExecuterOptions } -// InputProvider is an input provider interface for the nuclei execution +// InputProvider is an input providing interface for the nuclei execution // engine. // -// An example InputProvider is provided in form of hmap input provider. +// An example InputProvider implementation is provided in form of hybrid +// input provider in pkg/core/inputs/hybrid/hmap.go type InputProvider interface { // Count returns the number of items for input provider Count() int64 - // Scan calls a callback function till the input provider is exhausted + // Scan iterates the input and for each found item calls the callback + // until the input provider has been exhausted. Scan(callback func(value string)) } diff --git a/v2/pkg/parsers/workflow_loader.go b/v2/pkg/parsers/workflow_loader.go index e3f9ab8b..e0efd3fe 100644 --- a/v2/pkg/parsers/workflow_loader.go +++ b/v2/pkg/parsers/workflow_loader.go @@ -18,7 +18,7 @@ func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error) tagFilter := filter.New(&filter.Config{ Tags: options.Options.Tags, ExcludeTags: options.Options.ExcludeTags, - Authors: options.Options.Author, + Authors: options.Options.Authors, Severities: options.Options.Severities, IncludeTags: options.Options.IncludeTags, }) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 7f5a1851..f618484e 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -29,8 +29,8 @@ type Options struct { Severities severity.Severities // ExcludeSeverities specifies severities to exclude ExcludeSeverities severity.Severities - // Author filters templates based on their author and only run the matching ones. - Author goflags.NormalizedStringSlice + // Authors filters templates based on their author and only run the matching ones. + Authors goflags.NormalizedStringSlice // IncludeTags includes specified tags to be run even while being in denylist IncludeTags goflags.NormalizedStringSlice // IncludeTemplates includes specified templates to be run even while being in denylist From cf7628c45037dff4777f831d9ae1c02ed6d6724f Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 3 Nov 2021 02:34:48 +0530 Subject: [PATCH 078/146] Misc changes according to review --- v2/internal/testutils/integration.go | 12 +- v2/pkg/protocols/others/others.go | 1 - .../protocols/others/websocket/websocket.go | 365 ------------------ v2/pkg/protocols/{others => }/ssl/ssl.go | 62 +-- v2/pkg/protocols/{others => }/ssl/ssl_test.go | 0 v2/pkg/templates/templates.go | 4 +- 6 files changed, 36 insertions(+), 408 deletions(-) delete mode 100644 v2/pkg/protocols/others/others.go delete mode 100644 v2/pkg/protocols/others/websocket/websocket.go rename v2/pkg/protocols/{others => }/ssl/ssl.go (65%) rename v2/pkg/protocols/{others => }/ssl/ssl_test.go (100%) diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index 4c3f84e3..57fa1e0d 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -137,16 +137,10 @@ func NewWebsocketServer(path string, handler func(conn net.Conn), originValidate }() }) - var router *httprouter.Router if path != "" { - router = httprouter.New() + router := httprouter.New() router.HandlerFunc("*", "/test", handlerFunc) + return httptest.NewServer(router) } - var ts *httptest.Server - if router != nil { - ts = httptest.NewServer(router) - } else { - ts = httptest.NewServer(handlerFunc) - } - return ts + return httptest.NewServer(handlerFunc) } diff --git a/v2/pkg/protocols/others/others.go b/v2/pkg/protocols/others/others.go deleted file mode 100644 index 24b5bd24..00000000 --- a/v2/pkg/protocols/others/others.go +++ /dev/null @@ -1 +0,0 @@ -package others diff --git a/v2/pkg/protocols/others/websocket/websocket.go b/v2/pkg/protocols/others/websocket/websocket.go deleted file mode 100644 index ec73e827..00000000 --- a/v2/pkg/protocols/others/websocket/websocket.go +++ /dev/null @@ -1,365 +0,0 @@ -package websocket - -import ( - "context" - "crypto/tls" - "io" - "net/http" - "net/url" - "strings" - "time" - - "github.com/gobwas/ws" - "github.com/gobwas/ws/wsutil" - "github.com/pkg/errors" - "github.com/projectdiscovery/fastdialer/fastdialer" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/operators" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" - "github.com/projectdiscovery/nuclei/v2/pkg/types" -) - -// Request is a request for the Websocket protocol -type Request struct { - // Operators for the current request go here. - operators.Operators `yaml:",inline,omitempty"` - CompiledOperators *operators.Operators `yaml:"-"` - - // description: | - // Address contains address for the request - Address string `yaml:"address,omitempty" jsonschema:"title=address for the websocket request,description=Address contains address for the request"` - // description: | - // Inputs contains inputs for the websocket protocol - Inputs []*Input `yaml:"inputs,omitempty" jsonschema:"title=inputs for the websocket request,description=Inputs contains any input/output for the current request"` - // description: | - // Headers contains headers for the request. - Headers map[string]string `yaml:"headers,omitempty" jsonschema:"title=headers contains the request headers,description=Headers contains headers for the request"` - - // description: | - // Attack is the type of payload combinations to perform. - // - // Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates - // permutations and combinations for all payloads. - // values: - // - "sniper" - // - "pitchfork" - // - "clusterbomb" - AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` - // description: | - // Payloads contains any payloads for the current request. - // - // Payloads support both key-values combinations where a list - // of payloads is provided, or optionally a single file can also - // be provided as payload which will be read on run-time. - Payloads map[string]interface{} `yaml:"payloads,omitempty" jsonschema:"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request"` - - generator *generators.Generator - attackType generators.Type - - // cache any variables that may be needed for operation. - dialer *fastdialer.Dialer - options *protocols.ExecuterOptions -} - -// Input is an input for the websocket protocol -type Input struct { - // description: | - // Data is the data to send as the input. - // - // It supports DSL Helper Functions as well as normal expressions. - // examples: - // - value: "\"TEST\"" - // - value: "\"hex_decode('50494e47')\"" - Data string `yaml:"data,omitempty" jsonschema:"title=data to send as input,description=Data is the data to send as the input"` - // description: | - // Name is the optional name of the data read to provide matching on. - // examples: - // - value: "\"prefix\"" - Name string `yaml:"name,omitempty" jsonschema:"title=optional name for data read,description=Optional name of the data read to provide matching on"` -} - -// Compile compiles the request generators preparing any requests possible. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { - r.options = options - - client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) - if err != nil { - return errors.Wrap(err, "could not get network client") - } - r.dialer = client - - if len(r.Payloads) > 0 { - attackType := r.AttackType - if attackType == "" { - attackType = "batteringram" - } - r.attackType = generators.StringToType[attackType] - - // Resolve payload paths if they are files. - for name, payload := range r.Payloads { - payloadStr, ok := payload.(string) - if ok { - final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) - if resolveErr != nil { - return errors.Wrap(resolveErr, "could not read payload file") - } - r.Payloads[name] = final - } - } - r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath) - if err != nil { - return errors.Wrap(err, "could not parse payloads") - } - } - - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators - if err := compiled.Compile(); err != nil { - return errors.Wrap(err, "could not compile operators") - } - r.CompiledOperators = compiled - } - return nil -} - -// Requests returns the total number of requests the rule will perform -func (r *Request) Requests() int { - if r.generator != nil { - return r.generator.NewIterator().Total() - } - return 1 -} - -// GetID returns the ID for the request if any. -func (r *Request) GetID() string { - return "" -} - -// ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - hostname, err := getAddress(input) - if err != nil { - return nil - } - - if r.generator != nil { - iterator := r.generator.NewIterator() - - for { - value, ok := iterator.Value() - if !ok { - break - } - if err := r.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { - return err - } - } - } else { - value := make(map[string]interface{}) - if err := r.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { - return err - } - } - return nil -} - -// ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - header := http.Header{} - - payloadValues := make(map[string]interface{}) - for k, v := range dynamicValues { - payloadValues[k] = v - } - parsed, err := url.Parse(input) - if err != nil { - return errors.Wrap(err, "could not parse input url") - } - payloadValues["Hostname"] = parsed.Host - payloadValues["Host"] = parsed.Hostname() - payloadValues["Scheme"] = parsed.Scheme - payloadValues["Path"] = parsed.Path - - for key, value := range r.Headers { - finalData, dataErr := expressions.EvaluateByte([]byte(value), payloadValues) - if dataErr != nil { - r.options.Output.Request(r.options.TemplateID, input, "websocket", dataErr) - r.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(dataErr, "could not evaluate template expressions") - } - header.Set(key, string(finalData)) - } - websocketDialer := ws.Dialer{ - Header: ws.HandshakeHeaderHTTP(header), - Timeout: time.Duration(r.options.Options.Timeout) * time.Second, - NetDial: r.dialer.Dial, - TLSConfig: &tls.Config{InsecureSkipVerify: true, ServerName: hostname}, - } - - finalAddress, dataErr := expressions.EvaluateByte([]byte(r.Address), payloadValues) - if dataErr != nil { - r.options.Output.Request(r.options.TemplateID, input, "websocket", dataErr) - r.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(dataErr, "could not evaluate template expressions") - } - addressToDial := string(finalAddress) - if parsed.Path != "" && parsed.Path != "/" { - addressToDial = addressToDial + parsed.Path - } - - conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial) - if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "websocket", err) - r.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(err, "could not connect to server") - } - defer conn.Close() - - responseBuilder := &strings.Builder{} - if readBuffer != nil { - _, _ = io.Copy(responseBuilder, readBuffer) // Copy initial response - } - - reqBuilder := &strings.Builder{} - - inputEvents := make(map[string]interface{}) - for _, req := range r.Inputs { - reqBuilder.Grow(len(req.Data)) - - finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues) - if dataErr != nil { - r.options.Output.Request(r.options.TemplateID, input, "websocket", dataErr) - r.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(dataErr, "could not evaluate template expressions") - } - reqBuilder.WriteString(string(finalData)) - - err = wsutil.WriteClientMessage(conn, ws.OpText, finalData) - if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "websocket", err) - r.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(err, "could not write request to server") - } - - msg, _, err := wsutil.ReadServerData(conn) - if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "websocket", err) - r.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(err, "could not write request to server") - } - - responseBuilder.Write(msg) - if req.Name != "" { - bufferStr := string(msg) - if req.Name != "" { - inputEvents[req.Name] = bufferStr - } - - // Run any internal extractors for the request here and add found values to map. - if r.CompiledOperators != nil { - values := r.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{req.Name: bufferStr}, protocols.MakeDefaultExtractFunc) - for k, v := range values { - dynamicValues[k] = v - } - } - } - } - r.options.Progress.IncrementRequests() - - if r.options.Options.Debug || r.options.Options.DebugRequests { - requestOutput := reqBuilder.String() - gologger.Info().Str("address", input).Msgf("[%s] Dumped Websocket request for %s", r.options.TemplateID, input) - gologger.Print().Msgf("%s", requestOutput) - } - - r.options.Output.Request(r.options.TemplateID, input, "websocket", err) - gologger.Verbose().Msgf("Sent Websocket request to %s", input) - - data := make(map[string]interface{}) - for k, v := range previous { - data[k] = v - } - for k, v := range dynamicValues { - data[k] = v - } - for k, v := range inputEvents { - data[k] = v - } - data["success"] = "true" - data["request"] = reqBuilder.String() - data["response"] = responseBuilder.String() - data["host"] = input - data["ip"] = r.dialer.GetDialedIP(hostname) - - event := eventcreator.CreateEventWithAdditionalOptions(r, data, r.options.Options.Debug || r.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { - internalWrappedEvent.OperatorsResult.PayloadValues = dynamicValues - }) - if r.options.Options.Debug || r.options.Options.DebugResponse { - responseOutput := responseBuilder.String() - gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", r.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, r.options.Options.NoColor)) - } - - callback(event) - return nil -} - -// getAddress returns the address of the host to make request to -func getAddress(toTest string) (string, error) { - if !strings.HasPrefix(toTest, "ws://") && !strings.HasPrefix(toTest, "wss://") { - return "", errors.New("invalid websocket provided") - } - parsed, _ := url.Parse(toTest) - if parsed != nil && parsed.Host != "" { - return parsed.Host, nil - } - return "", nil -} - -// Match performs matching operation for a matcher on model and returns: -// true and a list of matched snippets if the matcher type is supports it -// otherwise false and an empty string slice -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - return protocols.MakeDefaultMatchFunc(data, matcher) -} - -// Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { - return protocols.MakeDefaultExtractFunc(data, matcher) -} - -// MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - return protocols.MakeDefaultResultEvent(r, wrapped) -} - -// GetCompiledOperators returns a list of the compiled operators -func (r *Request) GetCompiledOperators() []*operators.Operators { - return []*operators.Operators{r.CompiledOperators} -} - -func (r *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { - data := &output.ResultEvent{ - TemplateID: types.ToString(r.options.TemplateID), - TemplatePath: types.ToString(r.options.TemplatePath), - Info: r.options.TemplateInfo, - Type: "websocket", - Host: types.ToString(wrapped.InternalEvent["host"]), - Matched: types.ToString(wrapped.InternalEvent["host"]), - Metadata: wrapped.OperatorsResult.PayloadValues, - ExtractedResults: wrapped.OperatorsResult.OutputExtracts, - Timestamp: time.Now(), - IP: types.ToString(wrapped.InternalEvent["ip"]), - Request: types.ToString(wrapped.InternalEvent["request"]), - Response: types.ToString(wrapped.InternalEvent["response"]), - } - return data -} diff --git a/v2/pkg/protocols/others/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go similarity index 65% rename from v2/pkg/protocols/others/ssl/ssl.go rename to v2/pkg/protocols/ssl/ssl.go index f8f34b00..3e9eb117 100644 --- a/v2/pkg/protocols/others/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -36,37 +36,37 @@ type Request struct { } // Compile compiles the request generators preparing any requests possible. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { - r.options = options +func (request *Request) Compile(options *protocols.ExecuterOptions) error { + request.options = options client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) if err != nil { return errors.Wrap(err, "could not get network client") } - r.dialer = client + request.dialer = client - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } return nil } // Requests returns the total number of requests the rule will perform -func (r *Request) Requests() int { +func (request *Request) Requests() int { return 1 } // GetID returns the ID for the request if any. -func (r *Request) GetID() string { +func (request *Request) GetID() string { return "" } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { address, err := getAddress(input) if err != nil { return nil @@ -75,24 +75,24 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu config := &tls.Config{InsecureSkipVerify: true, ServerName: hostname} - conn, err := r.dialer.DialTLSWithConfig(context.Background(), "tcp", address, config) + conn, err := request.dialer.DialTLSWithConfig(context.Background(), "tcp", address, config) if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "ssl", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "ssl", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server") } defer conn.Close() - _ = conn.SetReadDeadline(time.Now().Add(time.Duration(r.options.Options.Timeout) * time.Second)) + _ = conn.SetReadDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second)) connTLS, ok := conn.(*tls.Conn) if !ok { return nil } - r.options.Output.Request(r.options.TemplateID, address, "ssl", err) + request.options.Output.Request(request.options.TemplateID, address, "ssl", err) gologger.Verbose().Msgf("Sent SSL request to %s", address) - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Str("address", input).Msgf("[%s] Dumped SSL request for %s", r.options.TemplateID, input) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Str("address", input).Msgf("[%s] Dumped SSL request for %s", request.options.TemplateID, input) } state := connTLS.ConnectionState() @@ -110,13 +110,13 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu data["response"] = jsonDataString data["host"] = input data["not_after"] = float64(cert.NotAfter.Unix()) - data["ip"] = r.dialer.GetDialedIP(hostname) + data["ip"] = request.dialer.GetDialedIP(hostname) - event := eventcreator.CreateEvent(r, data, r.options.Options.Debug || r.options.Options.DebugResponse) - if r.options.Options.Debug || r.options.Options.DebugResponse { + event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse) + if request.options.Options.Debug || request.options.Options.DebugResponse { responseOutput := jsonDataString - gologger.Debug().Msgf("[%s] Dumped SSL response for %s", r.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, r.options.Options.NoColor)) + gologger.Debug().Msgf("[%s] Dumped SSL response for %s", request.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, request.options.Options.NoColor)) } callback(event) return nil @@ -144,30 +144,30 @@ func getAddress(toTest string) (string, error) { // Match performs matching operation for a matcher on model and returns: // true and a list of matched snippets if the matcher type is supports it // otherwise false and an empty string slice -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { return protocols.MakeDefaultMatchFunc(data, matcher) } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { return protocols.MakeDefaultExtractFunc(data, matcher) } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - return protocols.MakeDefaultResultEvent(r, wrapped) +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) } // GetCompiledOperators returns a list of the compiled operators -func (r *Request) GetCompiledOperators() []*operators.Operators { - return []*operators.Operators{r.CompiledOperators} +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} } -func (r *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ - TemplateID: types.ToString(r.options.TemplateID), - TemplatePath: types.ToString(r.options.TemplatePath), - Info: r.options.TemplateInfo, + TemplateID: types.ToString(request.options.TemplateID), + TemplatePath: types.ToString(request.options.TemplatePath), + Info: request.options.TemplateInfo, Type: "ssl", Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["host"]), diff --git a/v2/pkg/protocols/others/ssl/ssl_test.go b/v2/pkg/protocols/ssl/ssl_test.go similarity index 100% rename from v2/pkg/protocols/others/ssl/ssl_test.go rename to v2/pkg/protocols/ssl/ssl_test.go diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 79e6d766..7e7f05a9 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -9,8 +9,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/ssl" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/websocket" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/ssl" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/websocket" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) From 47949c0b522ce305aa50a924273809a468acd8d5 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 3 Nov 2021 17:18:35 +0530 Subject: [PATCH 079/146] Use separate type enum for protocol types --- v2/cmd/nuclei/main.go | 5 +- v2/internal/runner/runner.go | 4 +- v2/pkg/catalog/loader/filter/tag_filter.go | 26 +-- .../catalog/loader/filter/tag_filter_test.go | 43 ++--- v2/pkg/catalog/loader/loader.go | 9 +- v2/pkg/parsers/parser.go | 3 +- v2/pkg/templates/templates.go | 17 +- v2/pkg/templates/types/types.go | 154 ++++++++++++++++++ v2/pkg/types/types.go | 5 +- 9 files changed, 216 insertions(+), 50 deletions(-) create mode 100644 v2/pkg/templates/types/types.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 99ef6558..3d657024 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/runner" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -68,8 +69,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), - flagSet.NormalizedStringSliceVarP(&options.Protocols, "type", "tp", []string{}, "protocol types to be executed (http,dns,headless,network,file,etc)"), - flagSet.NormalizedStringSliceVarP(&options.ExcludeProtocols, "exclude-type", "etype", []string{}, "protocol types to not be executed (http,dns,file,etc"), + flagSet.VarP(&options.Protocols, "type", "tp", fmt.Sprintf("protocol types to be executed. Possible values: %s", templateTypes.GetSupportedProtocolTypes())), + flagSet.VarP(&options.ExcludeProtocols, "exclude-type", "etype", fmt.Sprintf("protocol types to not be executed. Possible values: %s", templateTypes.GetSupportedProtocolTypes())), flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index c4c550ea..78ed7f13 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -376,8 +376,8 @@ func (r *Runner) RunEnumeration() error { ExcludeSeverities: r.options.ExcludeSeverities, IncludeTags: r.options.IncludeTags, TemplatesDirectory: r.options.TemplatesDirectory, - Types: r.options.Protocols, - ExcludeTypes: r.options.ExcludeProtocols, + Protocols: r.options.Protocols, + ExcludeProtocols: r.options.ExcludeProtocols, Catalog: r.catalog, ExecutorOptions: executerOpts, } diff --git a/v2/pkg/catalog/loader/filter/tag_filter.go b/v2/pkg/catalog/loader/filter/tag_filter.go index c33e96fd..1420003e 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter.go +++ b/v2/pkg/catalog/loader/filter/tag_filter.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) // TagFilter is used to filter nuclei templates for tag based execution @@ -15,8 +16,8 @@ type TagFilter struct { authors map[string]struct{} block map[string]struct{} matchAllows map[string]struct{} - types map[string]struct{} - excludeTypes map[string]struct{} + types map[types.ProtocolType]struct{} + excludeTypes map[types.ProtocolType]struct{} } // ErrExcluded is returned for excluded templates @@ -27,7 +28,7 @@ var ErrExcluded = errors.New("the template was excluded") // unless it is explicitly specified by user using the includeTags (matchAllows field). // Matching rule: (tag1 OR tag2...) AND (author1 OR author2...) AND (severity1 OR severity2...) AND (extraTags1 OR extraTags2...) // Returns true if the template matches the filter criteria, false otherwise. -func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string, templateType string) (bool, error) { +func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string, templateType types.ProtocolType) (bool, error) { for _, templateTag := range templateTags { _, blocked := tagFilter.block[templateTag] _, allowed := tagFilter.matchAllows[templateTag] @@ -121,8 +122,11 @@ func isTagMatch(tagFilter *TagFilter, templateTags []string) bool { return false } -func isTemplateTypeMatch(tagFilter *TagFilter, templateType string) bool { - if (len(tagFilter.excludeTypes) == 0 && len(tagFilter.types) == 0) || templateType == "" { +func isTemplateTypeMatch(tagFilter *TagFilter, templateType types.ProtocolType) bool { + if len(tagFilter.excludeTypes) == 0 && len(tagFilter.types) == 0 { + return true + } + if templateType.String() == "" || templateType == types.InvalidProtocol { return true } @@ -146,8 +150,8 @@ type Config struct { Severities severity.Severities ExcludeSeverities severity.Severities IncludeTags []string - Types []string - ExcludeTypes []string + Protocols types.ProtocolTypes + ExcludeProtocols types.ProtocolTypes } // New returns a tag filter for nuclei tag based execution @@ -161,8 +165,8 @@ func New(config *Config) *TagFilter { excludeSeverities: make(map[severity.Severity]struct{}), block: make(map[string]struct{}), matchAllows: make(map[string]struct{}), - types: make(map[string]struct{}), - excludeTypes: make(map[string]struct{}), + types: make(map[types.ProtocolType]struct{}), + excludeTypes: make(map[types.ProtocolType]struct{}), } for _, tag := range config.ExcludeTags { for _, val := range splitCommaTrim(tag) { @@ -204,12 +208,12 @@ func New(config *Config) *TagFilter { delete(filter.block, val) } } - for _, tag := range config.Types { + for _, tag := range config.Protocols { if _, ok := filter.types[tag]; !ok { filter.types[tag] = struct{}{} } } - for _, tag := range config.ExcludeTypes { + for _, tag := range config.ExcludeProtocols { if _, ok := filter.excludeTypes[tag]; !ok { filter.excludeTypes[tag] = struct{}{} } diff --git a/v2/pkg/catalog/loader/filter/tag_filter_test.go b/v2/pkg/catalog/loader/filter/tag_filter_test.go index 7599c27b..0758f6d4 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter_test.go +++ b/v2/pkg/catalog/loader/filter/tag_filter_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) func TestTagBasedFilter(t *testing.T) { @@ -15,19 +16,19 @@ func TestTagBasedFilter(t *testing.T) { }) t.Run("true", func(t *testing.T) { - matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, "") + matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") }) t.Run("false", func(t *testing.T) { - matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil, "") + matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) t.Run("match-extra-tags-positive", func(t *testing.T) { - matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, "") + matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol) require.True(t, matched, "could not get correct match") }) t.Run("match-extra-tags-negative", func(t *testing.T) { - matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, "") + matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) } @@ -36,7 +37,7 @@ func TestTagBasedFilter(t *testing.T) { filter := New(&Config{ ExcludeTags: []string{"dos"}, }) - matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil, "") + matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") require.Equal(t, ErrExcluded, err, "could not get correct error") }) @@ -46,7 +47,7 @@ func TestTagBasedFilter(t *testing.T) { ExcludeTags: []string{"dos", "fuzz"}, IncludeTags: []string{"fuzz"}, }) - matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") + matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.Nil(t, err, "could not get match") require.True(t, matched, "could not get correct match") }) @@ -55,7 +56,7 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"fuzz"}, ExcludeTags: []string{"fuzz"}, }) - matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") + matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.Nil(t, err, "could not get match") require.True(t, matched, "could not get correct match") }) @@ -63,24 +64,24 @@ func TestTagBasedFilter(t *testing.T) { filter := New(&Config{ Authors: []string{"pdteam"}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") }) t.Run("match-severity", func(t *testing.T) { filter := New(&Config{ Severities: severity.Severities{severity.High}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "") + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") }) t.Run("match-exclude-severity", func(t *testing.T) { filter := New(&Config{ ExcludeSeverities: severity.Severities{severity.Low}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "") + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") + matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) t.Run("match-exclude-with-tags", func(t *testing.T) { @@ -88,7 +89,7 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"tag"}, ExcludeTags: []string{"another"}, }) - matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil, "") + matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) t.Run("match-conditions", func(t *testing.T) { @@ -97,33 +98,33 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"jira"}, Severities: severity.Severities{severity.High}, }) - matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil, "") + matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, "") + matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil, "") + matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil, "") + matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) t.Run("match-type", func(t *testing.T) { filter := New(&Config{ - Types: []string{"http"}, + Protocols: []types.ProtocolType{types.HTTPProtocol}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "http") + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") }) t.Run("match-exclude-type", func(t *testing.T) { filter := New(&Config{ - ExcludeTypes: []string{"http"}, + ExcludeProtocols: []types.ProtocolType{types.HTTPProtocol}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "dns") + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.DNSProtocol) require.True(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "http") + matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) } diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index fe725e2e..604a7da5 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/parsers" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) // Config contains the configuration options for the loader @@ -21,8 +22,8 @@ type Config struct { Tags []string ExcludeTags []string - Types []string - ExcludeTypes []string + Protocols types.ProtocolTypes + ExcludeProtocols types.ProtocolTypes Authors []string Severities severity.Severities ExcludeSeverities severity.Severities @@ -58,8 +59,8 @@ func New(config *Config) (*Store, error) { Severities: config.Severities, ExcludeSeverities: config.ExcludeSeverities, IncludeTags: config.IncludeTags, - Types: config.Types, - ExcludeTypes: config.ExcludeTypes, + Protocols: config.Protocols, + ExcludeProtocols: config.ExcludeProtocols, }), pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{ IncludedTemplates: config.IncludeTemplates, diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 9e201fb0..e26e7488 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates/cache" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" ) @@ -59,7 +60,7 @@ func LoadWorkflow(templatePath string) (bool, error) { return false, nil } -func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string, templateType string) (bool, error) { +func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string, templateType types.ProtocolType) (bool, error) { templateTags := templateInfo.Tags.ToSlice() templateAuthors := templateInfo.Authors.ToSlice() templateSeverity := templateInfo.SeverityHolder.Severity diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 1e84def6..ff9e527f 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) @@ -75,19 +76,21 @@ type Template struct { } // Type returns the type of the template -func (t *Template) Type() string { +func (t *Template) Type() types.ProtocolType { switch { case len(t.RequestsDNS) > 0: - return "dns" + return types.DNSProtocol case len(t.RequestsFile) > 0: - return "file" + return types.FileProtocol case len(t.RequestsHTTP) > 0: - return "http" + return types.HTTPProtocol case len(t.RequestsHeadless) > 0: - return "headless" + return types.HeadlessProtocol case len(t.RequestsNetwork) > 0: - return "network" + return types.NetworkProtocol + case t.CompiledWorkflow != nil: + return types.WorkflowProtocol default: - return "" + return types.InvalidProtocol } } diff --git a/v2/pkg/templates/types/types.go b/v2/pkg/templates/types/types.go new file mode 100644 index 00000000..f82bac5b --- /dev/null +++ b/v2/pkg/templates/types/types.go @@ -0,0 +1,154 @@ +package types + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/alecthomas/jsonschema" + "github.com/pkg/errors" + "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" +) + +// ProtocolType is the type of the request protocol specified +type ProtocolType int + +// Supported values for the ProtocolType +const ( + DNSProtocol ProtocolType = iota + 1 + FileProtocol + HTTPProtocol + HeadlessProtocol + NetworkProtocol + WorkflowProtocol + limit + InvalidProtocol +) + +// ExtractorTypes is a table for conversion of extractor type from string. +var protocolMappings = map[ProtocolType]string{ + InvalidProtocol: "invalid", + DNSProtocol: "dns", + FileProtocol: "file", + HTTPProtocol: "http", + HeadlessProtocol: "headless", + NetworkProtocol: "network", + WorkflowProtocol: "workflow", +} + +func GetSupportedProtocolTypes() ProtocolTypes { + var result []ProtocolType + for index := ProtocolType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toProtocolType(valueToMap string) (ProtocolType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range protocolMappings { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("Invalid protocol type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToLower(value)) +} + +func (t ProtocolType) String() string { + return protocolMappings[t] +} + +// TypeHolder is used to hold internal type of the protocol +type TypeHolder struct { + ProtocolType ProtocolType +} + +func (holder TypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "type of the protocol", + Description: "Type of the protocol", + } + for _, types := range GetSupportedProtocolTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *TypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toProtocolType(marshalledTypes) + if err != nil { + return err + } + + holder.ProtocolType = computedType + return nil +} + +func (holder *TypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.ProtocolType.String()) +} + +func (holder TypeHolder) MarshalYAML() (interface{}, error) { + return holder.ProtocolType.String(), nil +} + +type ProtocolTypes []ProtocolType + +func (protocolTypes *ProtocolTypes) Set(values string) error { + inputTypes, err := goflags.ToNormalizedStringSlice(values) + if err != nil { + return err + } + + for _, inputType := range inputTypes { + if err := setProtocolType(protocolTypes, inputType); err != nil { + return err + } + } + return nil +} + +func (protocolTypes *ProtocolTypes) UnmarshalYAML(unmarshal func(interface{}) error) error { + var stringSliceValue stringslice.StringSlice + if err := unmarshal(&stringSliceValue); err != nil { + return err + } + + stringSLice := stringSliceValue.ToSlice() + var result = make(ProtocolTypes, 0, len(stringSLice)) + for _, typeString := range stringSLice { + if err := setProtocolType(&result, typeString); err != nil { + return err + } + } + *protocolTypes = result + return nil +} + +func (protocolTypes ProtocolTypes) String() string { + var stringTypes []string + for _, t := range protocolTypes { + stringTypes = append(stringTypes, t.String()) + } + return strings.Join(stringTypes, ", ") +} + +func setProtocolType(protocolTypes *ProtocolTypes, value string) error { + computedType, err := toProtocolType(value) + if err != nil { + return fmt.Errorf("'%s' is not a valid extract type", value) + } + *protocolTypes = append(*protocolTypes, computedType) + return nil +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 36f66503..fc65e3a1 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -3,6 +3,7 @@ package types import ( "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) // Options contains the configuration options for nuclei scanner. @@ -30,9 +31,9 @@ type Options struct { // ExcludeSeverities specifies severities to exclude ExcludeSeverities severity.Severities // Protocols contains the protocols to be allowed executed - Protocols goflags.NormalizedStringSlice + Protocols types.ProtocolTypes // ExcludeProtocols contains protocols to not be executed - ExcludeProtocols goflags.NormalizedStringSlice + ExcludeProtocols types.ProtocolTypes // Author filters templates based on their author and only run the matching ones. Author goflags.NormalizedStringSlice // IncludeTags includes specified tags to be run even while being in denylist From 4a0229c13d706860cd01fd1d7623762ea15e22b2 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 3 Nov 2021 18:36:54 +0530 Subject: [PATCH 080/146] Websocket protocol rename --- v2/pkg/protocols/websocket/websocket.go | 365 ++++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 v2/pkg/protocols/websocket/websocket.go diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go new file mode 100644 index 00000000..6aa90ede --- /dev/null +++ b/v2/pkg/protocols/websocket/websocket.go @@ -0,0 +1,365 @@ +package websocket + +import ( + "context" + "crypto/tls" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/gobwas/ws" + "github.com/gobwas/ws/wsutil" + "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Request is a request for the Websocket protocol +type Request struct { + // Operators for the current request go here. + operators.Operators `yaml:",inline,omitempty"` + CompiledOperators *operators.Operators `yaml:"-"` + + // description: | + // Address contains address for the request + Address string `yaml:"address,omitempty" jsonschema:"title=address for the websocket request,description=Address contains address for the request"` + // description: | + // Inputs contains inputs for the websocket protocol + Inputs []*Input `yaml:"inputs,omitempty" jsonschema:"title=inputs for the websocket request,description=Inputs contains any input/output for the current request"` + // description: | + // Headers contains headers for the request. + Headers map[string]string `yaml:"headers,omitempty" jsonschema:"title=headers contains the request headers,description=Headers contains headers for the request"` + + // description: | + // Attack is the type of payload combinations to perform. + // + // Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates + // permutations and combinations for all payloads. + // values: + // - "sniper" + // - "pitchfork" + // - "clusterbomb" + AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` + // description: | + // Payloads contains any payloads for the current request. + // + // Payloads support both key-values combinations where a list + // of payloads is provided, or optionally a single file can also + // be provided as payload which will be read on run-time. + Payloads map[string]interface{} `yaml:"payloads,omitempty" jsonschema:"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request"` + + generator *generators.Generator + attackType generators.Type + + // cache any variables that may be needed for operation. + dialer *fastdialer.Dialer + options *protocols.ExecuterOptions +} + +// Input is an input for the websocket protocol +type Input struct { + // description: | + // Data is the data to send as the input. + // + // It supports DSL Helper Functions as well as normal expressions. + // examples: + // - value: "\"TEST\"" + // - value: "\"hex_decode('50494e47')\"" + Data string `yaml:"data,omitempty" jsonschema:"title=data to send as input,description=Data is the data to send as the input"` + // description: | + // Name is the optional name of the data read to provide matching on. + // examples: + // - value: "\"prefix\"" + Name string `yaml:"name,omitempty" jsonschema:"title=optional name for data read,description=Optional name of the data read to provide matching on"` +} + +// Compile compiles the request generators preparing any requests possible. +func (request *Request) Compile(options *protocols.ExecuterOptions) error { + request.options = options + + client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) + if err != nil { + return errors.Wrap(err, "could not get network client") + } + request.dialer = client + + if len(request.Payloads) > 0 { + attackType := request.AttackType + if attackType == "" { + attackType = "batteringram" + } + request.attackType = generators.StringToType[attackType] + + // Resolve payload paths if they are files. + for name, payload := range request.Payloads { + payloadStr, ok := payload.(string) + if ok { + final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) + if resolveErr != nil { + return errors.Wrap(resolveErr, "could not read payload file") + } + request.Payloads[name] = final + } + } + request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath) + if err != nil { + return errors.Wrap(err, "could not parse payloads") + } + } + + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators + if err := compiled.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + request.CompiledOperators = compiled + } + return nil +} + +// Requests returns the total number of requests the rule will perform +func (request *Request) Requests() int { + if request.generator != nil { + return request.generator.NewIterator().Total() + } + return 1 +} + +// GetID returns the ID for the request if any. +func (request *Request) GetID() string { + return "" +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + hostname, err := getAddress(input) + if err != nil { + return nil + } + + if request.generator != nil { + iterator := request.generator.NewIterator() + + for { + value, ok := iterator.Value() + if !ok { + break + } + if err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { + return err + } + } + } else { + value := make(map[string]interface{}) + if err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { + return err + } + } + return nil +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (request *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + header := http.Header{} + + payloadValues := make(map[string]interface{}) + for k, v := range dynamicValues { + payloadValues[k] = v + } + parsed, err := url.Parse(input) + if err != nil { + return errors.Wrap(err, "could not parse input url") + } + payloadValues["Hostname"] = parsed.Host + payloadValues["Host"] = parsed.Hostname() + payloadValues["Scheme"] = parsed.Scheme + payloadValues["Path"] = parsed.Path + + for key, value := range request.Headers { + finalData, dataErr := expressions.EvaluateByte([]byte(value), payloadValues) + if dataErr != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not evaluate template expressions") + } + header.Set(key, string(finalData)) + } + websocketDialer := ws.Dialer{ + Header: ws.HandshakeHeaderHTTP(header), + Timeout: time.Duration(request.options.Options.Timeout) * time.Second, + NetDial: request.dialer.Dial, + TLSConfig: &tls.Config{InsecureSkipVerify: true, ServerName: hostname}, + } + + finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues) + if dataErr != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not evaluate template expressions") + } + addressToDial := string(finalAddress) + if parsed.Path != "" && parsed.Path != "/" { + addressToDial = addressToDial + parsed.Path + } + + conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial) + if err != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not connect to server") + } + defer conn.Close() + + responseBuilder := &strings.Builder{} + if readBuffer != nil { + _, _ = io.Copy(responseBuilder, readBuffer) // Copy initial response + } + + reqBuilder := &strings.Builder{} + + inputEvents := make(map[string]interface{}) + for _, req := range request.Inputs { + reqBuilder.Grow(len(req.Data)) + + finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues) + if dataErr != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not evaluate template expressions") + } + reqBuilder.WriteString(string(finalData)) + + err = wsutil.WriteClientMessage(conn, ws.OpText, finalData) + if err != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not write request to server") + } + + msg, _, err := wsutil.ReadServerData(conn) + if err != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not write request to server") + } + + responseBuilder.Write(msg) + if req.Name != "" { + bufferStr := string(msg) + if req.Name != "" { + inputEvents[req.Name] = bufferStr + } + + // Run any internal extractors for the request here and add found values to map. + if request.CompiledOperators != nil { + values := request.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{req.Name: bufferStr}, protocols.MakeDefaultExtractFunc) + for k, v := range values { + dynamicValues[k] = v + } + } + } + } + request.options.Progress.IncrementRequests() + + if request.options.Options.Debug || request.options.Options.DebugRequests { + requestOutput := reqBuilder.String() + gologger.Info().Str("address", input).Msgf("[%s] Dumped Websocket request for %s", request.options.TemplateID, input) + gologger.Print().Msgf("%s", requestOutput) + } + + request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + gologger.Verbose().Msgf("Sent Websocket request to %s", input) + + data := make(map[string]interface{}) + for k, v := range previous { + data[k] = v + } + for k, v := range dynamicValues { + data[k] = v + } + for k, v := range inputEvents { + data[k] = v + } + data["success"] = "true" + data["request"] = reqBuilder.String() + data["response"] = responseBuilder.String() + data["host"] = input + data["ip"] = request.dialer.GetDialedIP(hostname) + + event := eventcreator.CreateEventWithAdditionalOptions(request, data, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + internalWrappedEvent.OperatorsResult.PayloadValues = dynamicValues + }) + if request.options.Options.Debug || request.options.Options.DebugResponse { + responseOutput := responseBuilder.String() + gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", request.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, request.options.Options.NoColor)) + } + + callback(event) + return nil +} + +// getAddress returns the address of the host to make request to +func getAddress(toTest string) (string, error) { + if !strings.HasPrefix(toTest, "ws://") && !strings.HasPrefix(toTest, "wss://") { + return "", errors.New("invalid websocket provided") + } + parsed, _ := url.Parse(toTest) + if parsed != nil && parsed.Host != "" { + return parsed.Host, nil + } + return "", nil +} + +// Match performs matching operation for a matcher on model and returns: +// true and a list of matched snippets if the matcher type is supports it +// otherwise false and an empty string slice +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + return protocols.MakeDefaultMatchFunc(data, matcher) +} + +// Extract performs extracting operation for an extractor on model and returns true or false. +func (request *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { + return protocols.MakeDefaultExtractFunc(data, matcher) +} + +// MakeResultEvent creates a result event from internal wrapped event +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) +} + +// GetCompiledOperators returns a list of the compiled operators +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ + TemplateID: types.ToString(request.options.TemplateID), + TemplatePath: types.ToString(request.options.TemplatePath), + Info: request.options.TemplateInfo, + Type: "websocket", + Host: types.ToString(wrapped.InternalEvent["host"]), + Matched: types.ToString(wrapped.InternalEvent["host"]), + Metadata: wrapped.OperatorsResult.PayloadValues, + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + Timestamp: time.Now(), + IP: types.ToString(wrapped.InternalEvent["ip"]), + Request: types.ToString(wrapped.InternalEvent["request"]), + Response: types.ToString(wrapped.InternalEvent["response"]), + } + return data +} From 645ae30a47b998f2ca17d69ec2bc75f594684bcd Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 3 Nov 2021 19:53:45 +0530 Subject: [PATCH 081/146] Moved to an enum for TemplateType in protocols --- v2/pkg/protocols/dns/dns.go | 6 +-- v2/pkg/protocols/dns/dns_test.go | 12 +++--- v2/pkg/protocols/dns/operators_test.go | 48 +++++++++++----------- v2/pkg/protocols/dns/request.go | 6 +++ v2/pkg/protocols/dns/request_test.go | 12 +++--- v2/pkg/protocols/file/request.go | 6 +++ v2/pkg/protocols/headless/request.go | 6 +++ v2/pkg/protocols/http/request.go | 6 +++ v2/pkg/protocols/network/request.go | 6 +++ v2/pkg/protocols/offlinehttp/request.go | 6 +++ v2/pkg/protocols/ssl/ssl.go | 8 +++- v2/pkg/protocols/websocket/websocket.go | 8 +++- v2/pkg/templates/templates_doc_examples.go | 10 ++--- 13 files changed, 94 insertions(+), 46 deletions(-) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 8aab88af..a42df7ef 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -32,7 +32,7 @@ type Request struct { // - value: "\"{{FQDN}}\"" Name string `yaml:"name,omitempty" jsonschema:"title=hostname to make dns request for,description=Name is the Hostname to make DNS request for"` // description: | - // Type is the type of DNS request to make. + // RequestType is the type of DNS request to make. // values: // - "A" // - "NS" @@ -43,7 +43,7 @@ type Request struct { // - "MX" // - "TXT" // - "AAAA" - Type string `yaml:"type,omitempty" jsonschema:"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"` + RequestType string `yaml:"type,omitempty" jsonschema:"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"` // description: | // Class is the class of the DNS request. // @@ -111,7 +111,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } request.class = classToInt(request.Class) request.options = options - request.question = questionTypeToInt(request.Type) + request.question = questionTypeToInt(request.RequestType) return nil } diff --git a/v2/pkg/protocols/dns/dns_test.go b/v2/pkg/protocols/dns/dns_test.go index 85b00bd9..d01f84bd 100644 --- a/v2/pkg/protocols/dns/dns_test.go +++ b/v2/pkg/protocols/dns/dns_test.go @@ -27,12 +27,12 @@ func TestDNSCompileMake(t *testing.T) { testutils.Init(options) const templateID = "testing-dns" request := &Request{ - Type: "A", - Class: "INET", - Retries: 5, - ID: templateID, - Recursion: false, - Name: "{{FQDN}}", + RequestType: "A", + Class: "INET", + Retries: 5, + ID: templateID, + Recursion: false, + Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index 6bb91b7e..0e4c6ff3 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -23,12 +23,12 @@ func TestResponseToDSLMap(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - Type: "A", - Class: "INET", - Retries: 5, - ID: templateID, - Recursion: false, - Name: "{{FQDN}}", + RequestType: "A", + Class: "INET", + Retries: 5, + ID: templateID, + Recursion: false, + Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -55,12 +55,12 @@ func TestDNSOperatorMatch(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - Type: "A", - Class: "INET", - Retries: 5, - ID: templateID, - Recursion: false, - Name: "{{FQDN}}", + RequestType: "A", + Class: "INET", + Retries: 5, + ID: templateID, + Recursion: false, + Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -166,12 +166,12 @@ func TestDNSOperatorExtract(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - Type: "A", - Class: "INET", - Retries: 5, - ID: templateID, - Recursion: false, - Name: "{{FQDN}}", + RequestType: "A", + Class: "INET", + Retries: 5, + ID: templateID, + Recursion: false, + Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -223,12 +223,12 @@ func TestDNSMakeResult(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - Type: "A", - Class: "INET", - Retries: 5, - ID: templateID, - Recursion: false, - Name: "{{FQDN}}", + RequestType: "A", + Class: "INET", + Retries: 5, + ID: templateID, + Recursion: false, + Name: "{{FQDN}}", Operators: operators.Operators{ Matchers: []*matchers.Matcher{{ Name: "test", diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index d063825f..05d4930d 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -12,10 +12,16 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) var _ protocols.Request = &Request{} +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.DNSProtocol +} + // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // Parse the URL and return domain if URL. diff --git a/v2/pkg/protocols/dns/request_test.go b/v2/pkg/protocols/dns/request_test.go index c77bffae..bc5f230b 100644 --- a/v2/pkg/protocols/dns/request_test.go +++ b/v2/pkg/protocols/dns/request_test.go @@ -20,12 +20,12 @@ func TestDNSExecuteWithResults(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - Type: "A", - Class: "INET", - Retries: 5, - ID: templateID, - Recursion: false, - Name: "{{FQDN}}", + RequestType: "A", + Class: "INET", + Retries: 5, + ID: templateID, + Recursion: false, + Name: "{{FQDN}}", Operators: operators.Operators{ Matchers: []*matchers.Matcher{{ Name: "test", diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index c7c3daee..ef650bab 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -15,10 +15,16 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) var _ protocols.Request = &Request{} +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.FileProtocol +} + // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { wg := sizedwaitgroup.New(request.options.Options.BulkSize) diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 1609a595..dc1d3467 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -12,10 +12,16 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) var _ protocols.Request = &Request{} +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.HeadlessProtocol +} + // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(inputURL string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { instance, err := request.options.Browser.NewInstance() diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index b11340bf..af5075f3 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -28,12 +28,18 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/stringsutil" ) const defaultMaxWorkers = 150 +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.HTTPProtocol +} + // executeRaceRequest executes race condition request for a URL func (request *Request) executeRaceRequest(reqURL string, previous output.InternalEvent, callback protocols.OutputEventCallback) error { var generatedRequests []*generatedRequest diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 76135085..ed52117f 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -21,10 +21,16 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) var _ protocols.Request = &Request{} +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.NetworkProtocol +} + // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { var address string diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index c9dbc1a8..10b77b59 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -15,12 +15,18 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) var _ protocols.Request = &Request{} const maxSize = 5 * 1024 * 1024 +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.HTTPProtocol +} + // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { wg := sizedwaitgroup.New(request.options.Options.BulkSize) diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index ec79f332..6335e879 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -21,6 +21,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -163,12 +164,17 @@ func (request *Request) GetCompiledOperators() []*operators.Operators { return []*operators.Operators{request.CompiledOperators} } +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.SSLProtocol +} + func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, - Type: "ssl", + Type: request.Type().String(), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["host"]), Metadata: wrapped.OperatorsResult.PayloadValues, diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index f17a3b11..076855da 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -25,6 +25,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -355,7 +356,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, - Type: "websocket", + Type: request.Type().String(), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["host"]), Metadata: wrapped.OperatorsResult.PayloadValues, @@ -367,3 +368,8 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent } return data } + +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.WebsocketProtocol +} diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index 84580ffe..b755de29 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -37,11 +37,11 @@ var ( _ = exampleNormalHTTPRequest exampleNormalDNSRequest = &dns.Request{ - Name: "{{FQDN}}", - Type: "CNAME", - Class: "inet", - Retries: 2, - Recursion: true, + Name: "{{FQDN}}", + RequestType: "CNAME", + Class: "inet", + Retries: 2, + Recursion: true, Operators: operators.Operators{ Extractors: []*extractors.Extractor{ {Type: "regex", Regex: []string{"ec2-[-\\d]+\\.compute[-\\d]*\\.amazonaws\\.com", "ec2-[-\\d]+\\.[\\w\\d\\-]+\\.compute[-\\d]*\\.amazonaws\\.com"}}, From 91a7b4df6aa2e71e7d17c7d2642a2979d8cf980d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 3 Nov 2021 20:08:11 +0530 Subject: [PATCH 082/146] Validate opcode before proceeding with websocket --- v2/pkg/protocols/websocket/websocket.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 076855da..1ae6bbaa 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -251,12 +251,17 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam return errors.Wrap(err, "could not write request to server") } - msg, _, err := wsutil.ReadServerData(conn) + msg, opCode, err := wsutil.ReadServerData(conn) if err != nil { request.options.Output.Request(request.options.TemplateID, input, "websocket", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } + // Only perform matching and writes in case we recieve + // text or binary opcode from the websocket server. + if opCode != ws.OpText && opCode != ws.OpBinary { + continue + } responseBuilder.Write(msg) if req.Name != "" { From a7c8d0473c1f94692d4d69b2d8b7611db3db8790 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 4 Nov 2021 02:28:48 +0530 Subject: [PATCH 083/146] Removed duplicate attackType logic + move attackType to generators --- .../protocols/common/generators/generators.go | 39 ++++++++++++++++--- .../common/generators/generators_test.go | 10 +++-- v2/pkg/protocols/http/http.go | 24 +----------- .../protocols/http/request_generator_test.go | 11 ++++-- v2/pkg/protocols/network/network.go | 27 +------------ v2/pkg/protocols/websocket/websocket.go | 22 +---------- 6 files changed, 53 insertions(+), 80 deletions(-) diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index a63fbcb2..15ac1513 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -2,7 +2,12 @@ package generators -import "github.com/pkg/errors" +import ( + "fmt" + + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" +) // Generator is the generator struct for generating payloads type Generator struct { @@ -30,21 +35,45 @@ var StringToType = map[string]Type{ } // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, payloadType Type, templatePath string) (*Generator, error) { +func New(payloads map[string]interface{}, attackType, templatePath string, catalog *catalog.Catalog) (*Generator, error) { + if attackType == "" { + attackType = "batteringram" + } + attackTypeValue, ok := StringToType[attackType] + if !ok { + return nil, fmt.Errorf("invalid attack type provided: %s", attackType) + } + + // Resolve payload paths if they are files. + payloadsFinal := make(map[string]interface{}) + for name, payload := range payloads { + payloadsFinal[name] = payload + } + for name, payload := range payloads { + payloadStr, ok := payload.(string) + if ok { + final, resolveErr := catalog.ResolvePath(payloadStr, templatePath) + if resolveErr != nil { + return nil, errors.Wrap(resolveErr, "could not read payload file") + } + payloadsFinal[name] = final + } + } + generator := &Generator{} if err := generator.validate(payloads, templatePath); err != nil { return nil, err } - compiled, err := loadPayloads(payloads) + compiled, err := loadPayloads(payloadsFinal) if err != nil { return nil, err } - generator.Type = payloadType + generator.Type = attackTypeValue generator.payloads = compiled // Validate the batteringram payload set - if payloadType == BatteringRam { + if attackTypeValue == BatteringRam { if len(payloads) != 1 { return nil, errors.New("batteringram must have single payload set") } diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index de37ca62..8f6d9893 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -3,13 +3,15 @@ package generators import ( "testing" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/stretchr/testify/require" ) func TestBatteringRamGenerator(t *testing.T) { usernames := []string{"admin", "password"} - generator, err := New(map[string]interface{}{"username": usernames}, BatteringRam, "") + catalogInstance := catalog.New("") + generator, err := New(map[string]interface{}{"username": usernames}, "batteringram", "", catalogInstance) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -28,7 +30,8 @@ func TestPitchforkGenerator(t *testing.T) { usernames := []string{"admin", "token"} passwords := []string{"password1", "password2", "password3"} - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchFork, "") + catalogInstance := catalog.New("") + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, "pitchfork", "", catalogInstance) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -49,7 +52,8 @@ func TestClusterbombGenerator(t *testing.T) { usernames := []string{"admin"} passwords := []string{"admin", "password", "token"} - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBomb, "") + catalogInstance := catalog.New("") + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, "clusterbomb", "", catalogInstance) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 304f1a49..1c71ea5f 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -129,7 +129,6 @@ type Request struct { CompiledOperators *operators.Operators `yaml:"-"` options *protocols.ExecuterOptions - attackType generators.Type totalRequests int customHeaders map[string]string generator *generators.Generator // optional, only enabled when using payloads @@ -267,28 +266,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } if len(request.Payloads) > 0 { - attackType := request.AttackType - if attackType == "" { - attackType = "batteringram" - } - var ok bool - request.attackType, ok = generators.StringToType[attackType] - if !ok { - return fmt.Errorf("invalid attack type provided: %s", attackType) - } - - // Resolve payload paths if they are files. - for name, payload := range request.Payloads { - payloadStr, ok := payload.(string) - if ok { - final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) - if resolveErr != nil { - return errors.Wrap(resolveErr, "could not read payload file") - } - request.Payloads[name] = final - } - } - request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath) + request.generator, err = generators.New(request.Payloads, request.AttackType, request.options.TemplatePath, request.options.Catalog) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/http/request_generator_test.go b/v2/pkg/protocols/http/request_generator_test.go index 53064cca..c6ebec68 100644 --- a/v2/pkg/protocols/http/request_generator_test.go +++ b/v2/pkg/protocols/http/request_generator_test.go @@ -3,6 +3,7 @@ package http import ( "testing" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/stretchr/testify/require" ) @@ -28,10 +29,11 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - attackType: generators.ClusterBomb, + AttackType: "clusterbomb", Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, } - req.generator, err = generators.New(req.Payloads, req.attackType, "") + catalogInstance := catalog.New("") + req.generator, err = generators.New(req.Payloads, req.AttackType, "", catalogInstance) require.Nil(t, err, "could not create generator") generator := req.newGenerator() @@ -51,10 +53,11 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - attackType: generators.ClusterBomb, + AttackType: "clusterbomb", Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, } - req.generator, err = generators.New(req.Payloads, req.attackType, "") + catalogInstance := catalog.New("") + req.generator, err = generators.New(req.Payloads, req.AttackType, "", catalogInstance) require.Nil(t, err, "could not create generator") generator := req.newGenerator() diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 12094ae6..c0a0fa2e 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -1,7 +1,6 @@ package network import ( - "fmt" "net" "strings" @@ -76,8 +75,7 @@ type Request struct { operators.Operators `yaml:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-"` - generator *generators.Generator - attackType generators.Type + generator *generators.Generator // cache any variables that may be needed for operation. dialer *fastdialer.Dialer options *protocols.ExecuterOptions @@ -186,28 +184,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } if len(request.Payloads) > 0 { - attackType := request.AttackType - if attackType == "" { - attackType = "batteringram" - } - var ok bool - request.attackType, ok = generators.StringToType[attackType] - if !ok { - return fmt.Errorf("invalid attack type provided: %s", attackType) - } - - // Resolve payload paths if they are files. - for name, payload := range request.Payloads { - payloadStr, ok := payload.(string) - if ok { - final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) - if resolveErr != nil { - return errors.Wrap(resolveErr, "could not read payload file") - } - request.Payloads[name] = final - } - } - request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath) + request.generator, err = generators.New(request.Payloads, request.AttackType, request.options.TemplatePath, request.options.Catalog) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 1ae6bbaa..ffe6a0d2 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -63,8 +63,7 @@ type Request struct { // be provided as payload which will be read on run-time. Payloads map[string]interface{} `yaml:"payloads,omitempty" jsonschema:"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request"` - generator *generators.Generator - attackType generators.Type + generator *generators.Generator // cache any variables that may be needed for operation. dialer *fastdialer.Dialer @@ -99,24 +98,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { request.dialer = client if len(request.Payloads) > 0 { - attackType := request.AttackType - if attackType == "" { - attackType = "batteringram" - } - request.attackType = generators.StringToType[attackType] - - // Resolve payload paths if they are files. - for name, payload := range request.Payloads { - payloadStr, ok := payload.(string) - if ok { - final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) - if resolveErr != nil { - return errors.Wrap(resolveErr, "could not read payload file") - } - request.Payloads[name] = final - } - } - request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath) + request.generator, err = generators.New(request.Payloads, request.AttackType, request.options.TemplatePath, options.Catalog) if err != nil { return errors.Wrap(err, "could not parse payloads") } From a60b10afca45300759c3be636e4e0e9b5f487bff Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 4 Nov 2021 02:41:56 +0530 Subject: [PATCH 084/146] Added an enum for attackType --- .../common/generators/attack_types.go | 96 +++++++++++++++++++ .../protocols/common/generators/generators.go | 51 +++------- .../common/generators/generators_test.go | 6 +- v2/pkg/protocols/http/build_request_test.go | 5 +- v2/pkg/protocols/http/http.go | 4 +- v2/pkg/protocols/http/http_test.go | 3 +- .../protocols/http/request_generator_test.go | 8 +- v2/pkg/protocols/network/network.go | 4 +- v2/pkg/protocols/websocket/websocket.go | 4 +- 9 files changed, 127 insertions(+), 54 deletions(-) create mode 100644 v2/pkg/protocols/common/generators/attack_types.go diff --git a/v2/pkg/protocols/common/generators/attack_types.go b/v2/pkg/protocols/common/generators/attack_types.go new file mode 100644 index 00000000..ea445cbc --- /dev/null +++ b/v2/pkg/protocols/common/generators/attack_types.go @@ -0,0 +1,96 @@ +package generators + +import ( + "encoding/json" + "strings" + + "github.com/alecthomas/jsonschema" + "github.com/pkg/errors" +) + +// AttackType is the type of attack for payloads +type AttackType int + +// Supported values for the ProtocolType +const ( + // BatteringRamAttack replaces same payload into all of the defined payload positions at once. + BatteringRamAttack AttackType = iota + 1 + // PitchForkAttack replaces variables with positional value from multiple wordlists + PitchForkAttack + // ClusterbombAttack replaces variables with all possible combinations of values + ClusterbombAttack + limit +) + +// attackTypeMappings is a table for conversion of attack type from string. +var attackTypeMappings = map[AttackType]string{ + BatteringRamAttack: "batteringram", + PitchForkAttack: "pitchfork", + ClusterbombAttack: "clusterbomb", +} + +func GetSupportedAttackTypes() []AttackType { + var result []AttackType + for index := AttackType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toAttackType(valueToMap string) (AttackType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range attackTypeMappings { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("invalid attack type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToLower(value)) +} + +func (t AttackType) String() string { + return attackTypeMappings[t] +} + +// AttackTypeHolder is used to hold internal type of the protocol +type AttackTypeHolder struct { + Value AttackType +} + +func (holder AttackTypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "type of the protocol", + Description: "Type of the protocol", + } + for _, types := range GetSupportedAttackTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *AttackTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toAttackType(marshalledTypes) + if err != nil { + return err + } + + holder.Value = computedType + return nil +} + +func (holder *AttackTypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.Value.String()) +} + +func (holder AttackTypeHolder) MarshalYAML() (interface{}, error) { + return holder.Value.String(), nil +} diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 15ac1513..47b2fa44 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -3,45 +3,20 @@ package generators import ( - "fmt" - "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" ) // Generator is the generator struct for generating payloads type Generator struct { - Type Type + Type AttackType payloads map[string][]string } -// Type is type of attack -type Type int - -const ( - // Batteringram replaces same payload into all of the defined payload positions at once. - BatteringRam Type = iota + 1 - // PitchFork replaces variables with positional value from multiple wordlists - PitchFork - // ClusterBomb replaces variables with all possible combinations of values - ClusterBomb -) - -// StringToType is a table for conversion of attack type from string. -var StringToType = map[string]Type{ - "batteringram": BatteringRam, - "pitchfork": PitchFork, - "clusterbomb": ClusterBomb, -} - // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, attackType, templatePath string, catalog *catalog.Catalog) (*Generator, error) { - if attackType == "" { - attackType = "batteringram" - } - attackTypeValue, ok := StringToType[attackType] - if !ok { - return nil, fmt.Errorf("invalid attack type provided: %s", attackType) +func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog *catalog.Catalog) (*Generator, error) { + if attackType.String() == "" { + attackType = BatteringRamAttack } // Resolve payload paths if they are files. @@ -69,11 +44,11 @@ func New(payloads map[string]interface{}, attackType, templatePath string, catal if err != nil { return nil, err } - generator.Type = attackTypeValue + generator.Type = attackType generator.payloads = compiled // Validate the batteringram payload set - if attackTypeValue == BatteringRam { + if attackType == BatteringRamAttack { if len(payloads) != 1 { return nil, errors.New("batteringram must have single payload set") } @@ -83,7 +58,7 @@ func New(payloads map[string]interface{}, attackType, templatePath string, catal // Iterator is a single instance of an iterator for a generator structure type Iterator struct { - Type Type + Type AttackType position int msbIterator int total int @@ -124,18 +99,18 @@ func (i *Iterator) Remaining() int { func (i *Iterator) Total() int { count := 0 switch i.Type { - case BatteringRam: + case BatteringRamAttack: for _, p := range i.payloads { count += len(p.values) } - case PitchFork: + case PitchForkAttack: count = len(i.payloads[0].values) for _, p := range i.payloads { if count > len(p.values) { count = len(p.values) } } - case ClusterBomb: + case ClusterbombAttack: count = 1 for _, p := range i.payloads { count *= len(p.values) @@ -147,11 +122,11 @@ func (i *Iterator) Total() int { // Value returns the next value for an iterator func (i *Iterator) Value() (map[string]interface{}, bool) { switch i.Type { - case BatteringRam: + case BatteringRamAttack: return i.batteringRamValue() - case PitchFork: + case PitchForkAttack: return i.pitchforkValue() - case ClusterBomb: + case ClusterbombAttack: return i.clusterbombValue() default: return i.batteringRamValue() diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 8f6d9893..75e39d0e 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -11,7 +11,7 @@ func TestBatteringRamGenerator(t *testing.T) { usernames := []string{"admin", "password"} catalogInstance := catalog.New("") - generator, err := New(map[string]interface{}{"username": usernames}, "batteringram", "", catalogInstance) + generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", catalogInstance) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -31,7 +31,7 @@ func TestPitchforkGenerator(t *testing.T) { passwords := []string{"password1", "password2", "password3"} catalogInstance := catalog.New("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, "pitchfork", "", catalogInstance) + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", catalogInstance) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -53,7 +53,7 @@ func TestClusterbombGenerator(t *testing.T) { passwords := []string{"admin", "password", "token"} catalogInstance := catalog.New("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, "clusterbomb", "", catalogInstance) + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterbombAttack, "", catalogInstance) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index d45f7f8f..8443a7f6 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" ) func TestBaseURLWithTemplatePrefs(t *testing.T) { @@ -133,7 +134,7 @@ func TestMakeRequestFromRawWithPayloads(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: "clusterbomb", + AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) @@ -172,7 +173,7 @@ func TestMakeRequestFromRawPayloadExpressions(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: "clusterbomb", + AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 1c71ea5f..df79e8f4 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -52,7 +52,7 @@ type Request struct { // - "batteringram" // - "pitchfork" // - "clusterbomb" - AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` + AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` // description: | // Method is the HTTP Request Method. // values: @@ -266,7 +266,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType, request.options.TemplatePath, request.options.Catalog) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/http/http_test.go b/v2/pkg/protocols/http/http_test.go index 5f0e304b..b7da9da6 100644 --- a/v2/pkg/protocols/http/http_test.go +++ b/v2/pkg/protocols/http/http_test.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" ) func TestHTTPCompile(t *testing.T) { @@ -22,7 +23,7 @@ func TestHTTPCompile(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: "clusterbomb", + AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) diff --git a/v2/pkg/protocols/http/request_generator_test.go b/v2/pkg/protocols/http/request_generator_test.go index c6ebec68..272a6282 100644 --- a/v2/pkg/protocols/http/request_generator_test.go +++ b/v2/pkg/protocols/http/request_generator_test.go @@ -29,11 +29,11 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - AttackType: "clusterbomb", + AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, } catalogInstance := catalog.New("") - req.generator, err = generators.New(req.Payloads, req.AttackType, "", catalogInstance) + req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance) require.Nil(t, err, "could not create generator") generator := req.newGenerator() @@ -53,11 +53,11 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - AttackType: "clusterbomb", + AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, } catalogInstance := catalog.New("") - req.generator, err = generators.New(req.Payloads, req.AttackType, "", catalogInstance) + req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance) require.Nil(t, err, "could not create generator") generator := req.newGenerator() diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index c0a0fa2e..6b3cbe2a 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -40,7 +40,7 @@ type Request struct { // - "batteringram" // - "pitchfork" // - "clusterbomb" - AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` + AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` // description: | // Payloads contains any payloads for the current request. // @@ -184,7 +184,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType, request.options.TemplatePath, request.options.Catalog) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index ffe6a0d2..8079c777 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -54,7 +54,7 @@ type Request struct { // - "sniper" // - "pitchfork" // - "clusterbomb" - AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` + AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` // description: | // Payloads contains any payloads for the current request. // @@ -98,7 +98,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { request.dialer = client if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType, request.options.TemplatePath, options.Catalog) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, options.Catalog) if err != nil { return errors.Wrap(err, "could not parse payloads") } From 53164c3f3e1cdd4af471f1e813b5c2d4129c9a5b Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 4 Nov 2021 02:44:47 +0530 Subject: [PATCH 085/146] Misc linter related changes --- v2/pkg/protocols/common/generators/attack_types.go | 4 ++-- v2/pkg/protocols/http/build_request_test.go | 4 ++-- v2/pkg/protocols/http/http_test.go | 2 +- v2/pkg/protocols/http/request_generator_test.go | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/v2/pkg/protocols/common/generators/attack_types.go b/v2/pkg/protocols/common/generators/attack_types.go index ea445cbc..76902690 100644 --- a/v2/pkg/protocols/common/generators/attack_types.go +++ b/v2/pkg/protocols/common/generators/attack_types.go @@ -63,8 +63,8 @@ type AttackTypeHolder struct { func (holder AttackTypeHolder) JSONSchemaType() *jsonschema.Type { gotType := &jsonschema.Type{ Type: "string", - Title: "type of the protocol", - Description: "Type of the protocol", + Title: "type of the attack", + Description: "Type of the attack", } for _, types := range GetSupportedAttackTypes() { gotType.Enum = append(gotType.Enum, types.String()) diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index 8443a7f6..c5cc3aaf 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -134,7 +134,7 @@ func TestMakeRequestFromRawWithPayloads(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) @@ -173,7 +173,7 @@ func TestMakeRequestFromRawPayloadExpressions(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) diff --git a/v2/pkg/protocols/http/http_test.go b/v2/pkg/protocols/http/http_test.go index b7da9da6..69a71fb2 100644 --- a/v2/pkg/protocols/http/http_test.go +++ b/v2/pkg/protocols/http/http_test.go @@ -23,7 +23,7 @@ func TestHTTPCompile(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) diff --git a/v2/pkg/protocols/http/request_generator_test.go b/v2/pkg/protocols/http/request_generator_test.go index 272a6282..3fee8024 100644 --- a/v2/pkg/protocols/http/request_generator_test.go +++ b/v2/pkg/protocols/http/request_generator_test.go @@ -29,7 +29,7 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, } catalogInstance := catalog.New("") @@ -53,7 +53,7 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, } catalogInstance := catalog.New("") From 12321c23af3be5e199202066f5cf930f46a012c4 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 4 Nov 2021 02:56:59 +0530 Subject: [PATCH 086/146] Splitting big function into small --- v2/pkg/protocols/ssl/ssl.go | 3 +- v2/pkg/protocols/websocket/websocket.go | 113 +++++++++++++----------- 2 files changed, 61 insertions(+), 55 deletions(-) diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index 6335e879..8f624d5b 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -115,9 +115,8 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse) if request.options.Options.Debug || request.options.Options.DebugResponse { - responseOutput := jsonDataString gologger.Debug().Msgf("[%s] Dumped SSL response for %s", request.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, request.options.Options.NoColor, false)) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, request.options.Options.NoColor, false)) } callback(event) return nil diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 8079c777..4cea98d5 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "fmt" "io" + "net" "net/http" "net/url" "strings" @@ -212,57 +213,15 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam _, _ = io.Copy(responseBuilder, readBuffer) // Copy initial response } - reqBuilder := &strings.Builder{} - - inputEvents := make(map[string]interface{}) - for _, req := range request.Inputs { - reqBuilder.Grow(len(req.Data)) - - finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues) - if dataErr != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) - request.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(dataErr, "could not evaluate template expressions") - } - reqBuilder.WriteString(string(finalData)) - - err = wsutil.WriteClientMessage(conn, ws.OpText, finalData) - if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", err) - request.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(err, "could not write request to server") - } - - msg, opCode, err := wsutil.ReadServerData(conn) - if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", err) - request.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(err, "could not write request to server") - } - // Only perform matching and writes in case we recieve - // text or binary opcode from the websocket server. - if opCode != ws.OpText && opCode != ws.OpBinary { - continue - } - - responseBuilder.Write(msg) - if req.Name != "" { - bufferStr := string(msg) - inputEvents[req.Name] = bufferStr - - // Run any internal extractors for the request here and add found values to map. - if request.CompiledOperators != nil { - values := request.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{req.Name: bufferStr}, protocols.MakeDefaultExtractFunc) - for k, v := range values { - dynamicValues[k] = v - } - } - } + events, requestOutput, err := request.readWriteInputWebsocket(conn, payloadValues, input, responseBuilder) + if err != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not read write response") } request.options.Progress.IncrementRequests() if request.options.Options.Debug || request.options.Options.DebugRequests { - requestOutput := reqBuilder.String() gologger.Debug().Str("address", input).Msgf("[%s] Dumped Websocket request for %s", request.options.TemplateID, input) gologger.Print().Msgf("%s", requestOutput) } @@ -274,20 +233,17 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam for k, v := range previous { data[k] = v } - for k, v := range dynamicValues { - data[k] = v - } - for k, v := range inputEvents { + for k, v := range events { data[k] = v } data["success"] = "true" - data["request"] = reqBuilder.String() + data["request"] = requestOutput data["response"] = responseBuilder.String() data["host"] = input data["ip"] = request.dialer.GetDialedIP(hostname) event := eventcreator.CreateEventWithAdditionalOptions(request, data, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { - internalWrappedEvent.OperatorsResult.PayloadValues = dynamicValues + internalWrappedEvent.OperatorsResult.PayloadValues = payloadValues }) if request.options.Options.Debug || request.options.Options.DebugResponse { responseOutput := responseBuilder.String() @@ -299,6 +255,57 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam return nil } +func (request *Request) readWriteInputWebsocket(conn net.Conn, payloadValues map[string]interface{}, input string, respBuilder *strings.Builder) (events map[string]interface{}, req string, err error) { + reqBuilder := &strings.Builder{} + inputEvents := make(map[string]interface{}) + + for _, req := range request.Inputs { + reqBuilder.Grow(len(req.Data)) + + finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues) + if dataErr != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) + request.options.Progress.IncrementFailedRequestsBy(1) + return nil, "", errors.Wrap(dataErr, "could not evaluate template expressions") + } + reqBuilder.WriteString(string(finalData)) + + err = wsutil.WriteClientMessage(conn, ws.OpText, finalData) + if err != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + request.options.Progress.IncrementFailedRequestsBy(1) + return nil, "", errors.Wrap(err, "could not write request to server") + } + + msg, opCode, err := wsutil.ReadServerData(conn) + if err != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + request.options.Progress.IncrementFailedRequestsBy(1) + return nil, "", errors.Wrap(err, "could not write request to server") + } + // Only perform matching and writes in case we recieve + // text or binary opcode from the websocket server. + if opCode != ws.OpText && opCode != ws.OpBinary { + continue + } + + respBuilder.Write(msg) + if req.Name != "" { + bufferStr := string(msg) + inputEvents[req.Name] = bufferStr + + // Run any internal extractors for the request here and add found values to map. + if request.CompiledOperators != nil { + values := request.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{req.Name: bufferStr}, protocols.MakeDefaultExtractFunc) + for k, v := range values { + inputEvents[k] = v + } + } + } + } + return inputEvents, reqBuilder.String(), nil +} + // getAddress returns the address of the host to make request to func getAddress(toTest string) (string, error) { parsed, err := url.Parse(toTest) From f3675d547aeccfbdf9016b99e2cd422a4a2dd134 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 4 Nov 2021 03:02:29 +0530 Subject: [PATCH 087/146] URL parsing addition for websocket URLs --- v2/pkg/protocols/websocket/websocket.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 4cea98d5..7731dc29 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -8,6 +8,7 @@ import ( "net" "net/http" "net/url" + "path" "strings" "time" @@ -195,9 +196,18 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } + addressToDial := string(finalAddress) + parsedAddress, err := url.Parse(addressToDial) + if err != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not parse input url") + } + if parsed.Path != "" && parsed.Path != "/" { - addressToDial = addressToDial + parsed.Path + parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path) + addressToDial = parsedAddress.String() } conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial) @@ -240,6 +250,7 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam data["request"] = requestOutput data["response"] = responseBuilder.String() data["host"] = input + data["matched"] = addressToDial data["ip"] = request.dialer.GetDialedIP(hostname) event := eventcreator.CreateEventWithAdditionalOptions(request, data, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { @@ -352,7 +363,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Info: request.options.TemplateInfo, Type: request.Type().String(), Host: types.ToString(wrapped.InternalEvent["host"]), - Matched: types.ToString(wrapped.InternalEvent["host"]), + Matched: types.ToString(wrapped.InternalEvent["matched"]), Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), From 41393fc09dc6e24171e8854deb6fcefb827475ad Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Thu, 4 Nov 2021 17:13:47 +0530 Subject: [PATCH 088/146] unique payload per interactsh placeholder #1068 --- .../protocols/common/interactsh/interactsh.go | 55 ++++++++++--------- v2/pkg/protocols/http/build_request.go | 55 ++++++++----------- v2/pkg/protocols/http/build_request_test.go | 14 ++--- v2/pkg/protocols/http/request.go | 16 ++---- v2/pkg/protocols/http/request_generator.go | 1 + v2/pkg/protocols/network/request.go | 14 ++--- 6 files changed, 71 insertions(+), 84 deletions(-) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 5c2914ff..5fff7ac5 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -206,12 +206,14 @@ func (c *Client) Close() bool { // // It accepts data to replace as well as the URL to replace placeholders // with generated uniquely for each request. -func (c *Client) ReplaceMarkers(data, interactshURL string) string { - if !strings.Contains(data, interactshURLMarker) { - return data +func (c *Client) ReplaceMarkers(data string, interactshURLs []string) (string, []string) { + + for strings.Contains(data, interactshURLMarker) { + url := c.URL() + interactshURLs = append(interactshURLs, url) + data = strings.Replace(data, interactshURLMarker, url, 1) } - replaced := strings.NewReplacer("{{interactsh-url}}", interactshURL).Replace(data) - return replaced + return data, interactshURLs } // MakeResultEventFunc is a result making function for nuclei @@ -227,30 +229,33 @@ type RequestData struct { } // RequestEvent is the event for a network request sent by nuclei. -func (c *Client) RequestEvent(interactshURL string, data *RequestData) { - id := strings.TrimSuffix(interactshURL, c.dotHostname) +func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) { + for _, interactshURL := range interactshURLs { + id := strings.TrimSuffix(interactshURL, c.dotHostname) - interaction := c.interactions.Get(id) - if interaction != nil { - // If we have previous interactions, get them and process them. - interactions, ok := interaction.Value().([]*server.Interaction) - if !ok { - c.requests.Set(id, data, c.eviction) - return - } - matched := false - for _, interaction := range interactions { - if c.processInteractionForRequest(interaction, data) { - matched = true - break + interaction := c.interactions.Get(id) + if interaction != nil { + // If we have previous interactions, get them and process them. + interactions, ok := interaction.Value().([]*server.Interaction) + if !ok { + c.requests.Set(id, data, c.eviction) + return } + matched := false + for _, interaction := range interactions { + if c.processInteractionForRequest(interaction, data) { + matched = true + break + } + } + if matched { + c.interactions.Delete(id) + } + } else { + c.requests.Set(id, data, c.eviction) } - if matched { - c.interactions.Delete(id) - } - } else { - c.requests.Set(id, data, c.eviction) } + } // HasMatchers returns true if an operator has interactsh part diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index ea83aea7..20f3d480 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -38,6 +38,7 @@ type generatedRequest struct { pipelinedClient *rawhttp.PipelineClient request *retryablehttp.Request dynamicValues map[string]interface{} + interactshURLs []string } func (g *generatedRequest) URL() string { @@ -52,9 +53,9 @@ func (g *generatedRequest) URL() string { // Make creates a http request for the provided input. // It returns io.EOF as error when all the requests have been exhausted. -func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) { +func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}) (*generatedRequest, error) { if r.request.SelfContained { - return r.makeSelfContainedRequest(dynamicValues, interactURL) + return r.makeSelfContainedRequest(dynamicValues) } // We get the next payload for the request. data, payloads, ok := r.nextValue() @@ -63,12 +64,10 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa } ctx := context.Background() - if interactURL != "" { - data = r.options.Interactsh.ReplaceMarkers(data, interactURL) + data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) - for payloadName, payloadValue := range payloads { - payloads[payloadName] = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), interactURL) - } + for payloadName, payloadValue := range payloads { + payloads[payloadName], r.interactshURLs = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), r.interactshURLs) } parsed, err := url.Parse(baseURL) @@ -98,12 +97,12 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa // If data contains \n it's a raw request, process it like raw. Else // continue with the template based request flow. if isRawRequest { - return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL) + return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads) } - return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL) + return r.makeHTTPRequestFromModel(ctx, data, values, payloads) } -func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) { +func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]interface{}) (*generatedRequest, error) { // We get the next payload for the request. data, payloads, ok := r.nextValue() if !ok { @@ -136,13 +135,13 @@ func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]int generators.BuildPayloadFromOptions(r.request.options.Options), ) - return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL) + return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads) } values := generators.MergeMaps( dynamicValues, generators.BuildPayloadFromOptions(r.request.options.Options), ) - return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL) + return r.makeHTTPRequestFromModel(ctx, data, values, payloads) } // Total returns the total number of requests for the generator @@ -171,10 +170,8 @@ func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) { } // MakeHTTPRequestFromModel creates a *http.Request from a request template -func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values, generatorValues map[string]interface{}, interactURL string) (*generatedRequest, error) { - if interactURL != "" { - data = r.options.Interactsh.ReplaceMarkers(data, interactURL) - } +func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values, generatorValues map[string]interface{}) (*generatedRequest, error) { + data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) // Combine the template payloads along with base // request values. @@ -198,18 +195,16 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st return nil, err } - request, err := r.fillRequest(req, finalValues, interactURL) + request, err := r.fillRequest(req, finalValues) if err != nil { return nil, err } - return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues}, nil + return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues, interactshURLs: r.interactshURLs}, nil } // makeHTTPRequestFromRaw creates a *http.Request from a raw request -func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}, interactURL string) (*generatedRequest, error) { - if interactURL != "" { - data = r.options.Interactsh.ReplaceMarkers(data, interactURL) - } +func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}) (*generatedRequest, error) { + data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) return r.handleRawWithPayloads(ctx, data, baseURL, values, payloads) } @@ -258,21 +253,19 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest req.Host = value } } - request, err := r.fillRequest(req, finalValues, "") + request, err := r.fillRequest(req, finalValues) if err != nil { return nil, err } - return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues}, nil + return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues, interactshURLs: r.interactshURLs}, nil } // fillRequest fills various headers in the request with values -func (r *requestGenerator) fillRequest(req *http.Request, values map[string]interface{}, interactURL string) (*retryablehttp.Request, error) { +func (r *requestGenerator) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) { // Set the header values requested for header, value := range r.request.Headers { - if interactURL != "" { - value = r.options.Interactsh.ReplaceMarkers(value, interactURL) - } + value, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(value, r.interactshURLs) value, err := expressions.Evaluate(value, values) if err != nil { return nil, errors.Wrap(err, "could not evaluate helper expressions") @@ -290,10 +283,8 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte // Check if the user requested a request body if r.request.Body != "" { - body := r.request.Body - if interactURL != "" { - body = r.options.Interactsh.ReplaceMarkers(body, interactURL) - } + var body string + body, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(r.request.Body, r.interactshURLs) body, err := expressions.Evaluate(body, values) if err != nil { return nil, errors.Wrap(err, "could not evaluate helper expressions") diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index d45f7f8f..122d81bc 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -84,7 +84,7 @@ func TestMakeRequestFromModal(t *testing.T) { require.Nil(t, err, "could not compile http request") generator := request.newGenerator() - req, err := generator.Make("https://example.com", map[string]interface{}{}, "") + req, err := generator.Make("https://example.com", map[string]interface{}{}) require.Nil(t, err, "could not make http request") bodyBytes, _ := req.request.BodyBytes() @@ -111,12 +111,12 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) { require.Nil(t, err, "could not compile http request") generator := request.newGenerator() - req, err := generator.Make("https://example.com/test.php", map[string]interface{}{}, "") + req, err := generator.Make("https://example.com/test.php", map[string]interface{}{}) require.Nil(t, err, "could not make http request") require.Equal(t, "https://example.com/test.php?query=example", req.request.URL.String(), "could not get correct request path") generator = request.newGenerator() - req, err = generator.Make("https://example.com/test/", map[string]interface{}{}, "") + req, err = generator.Make("https://example.com/test/", map[string]interface{}{}) require.Nil(t, err, "could not make http request") require.Equal(t, "https://example.com/test/?query=example", req.request.URL.String(), "could not get correct request path") } @@ -149,12 +149,12 @@ Accept-Encoding: gzip`}, require.Nil(t, err, "could not compile http request") generator := request.newGenerator() - req, err := generator.Make("https://example.com", map[string]interface{}{}, "") + req, err := generator.Make("https://example.com", map[string]interface{}{}) require.Nil(t, err, "could not make http request") authorization := req.request.Header.Get("Authorization") require.Equal(t, "Basic admin:admin", authorization, "could not get correct authorization headers from raw") - req, err = generator.Make("https://example.com", map[string]interface{}{}, "") + req, err = generator.Make("https://example.com", map[string]interface{}{}) require.Nil(t, err, "could not make http request") authorization = req.request.Header.Get("Authorization") require.Equal(t, "Basic admin:guest", authorization, "could not get correct authorization headers from raw") @@ -188,12 +188,12 @@ Accept-Encoding: gzip`}, require.Nil(t, err, "could not compile http request") generator := request.newGenerator() - req, err := generator.Make("https://example.com", map[string]interface{}{}, "") + req, err := generator.Make("https://example.com", map[string]interface{}{}) require.Nil(t, err, "could not make http request") authorization := req.request.Header.Get("Authorization") require.Equal(t, "Basic YWRtaW46YWRtaW4=", authorization, "could not get correct authorization headers from raw") - req, err = generator.Make("https://example.com", map[string]interface{}{}, "") + req, err = generator.Make("https://example.com", map[string]interface{}{}) require.Nil(t, err, "could not make http request") authorization = req.request.Header.Get("Authorization") require.Equal(t, "Basic YWRtaW46Z3Vlc3Q=", authorization, "could not get correct authorization headers from raw") diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index b11340bf..c3472b45 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -41,7 +41,7 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern // Requests within race condition should be dumped once and the output prefilled to allow DSL language to work // This will introduce a delay and will populate in hacky way the field "request" of outputEvent generator := request.newGenerator() - requestForDump, err := generator.Make(reqURL, nil, "") + requestForDump, err := generator.Make(reqURL, nil) if err != nil { return err } @@ -59,7 +59,7 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern // Pre-Generate requests for i := 0; i < request.RaceNumberRequests; i++ { generator := request.newGenerator() - generatedRequest, err := generator.Make(reqURL, nil, "") + generatedRequest, err := generator.Make(reqURL, nil) if err != nil { return err } @@ -98,7 +98,7 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output. var requestErr error mutex := &sync.Mutex{} for { - generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "") + generatedHttpRequest, err := generator.Make(reqURL, dynamicValues) if err == io.EOF { break } @@ -161,7 +161,7 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous var requestErr error mutex := &sync.Mutex{} for { - generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "") + generatedHttpRequest, err := generator.Make(reqURL, dynamicValues) if err == io.EOF { break } @@ -215,11 +215,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou for { hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators) - var interactURL string - if request.options.Interactsh != nil && hasInteractMarkers { - interactURL = request.options.Interactsh.URL() - } - generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, interactURL) + generatedHttpRequest, err := generator.Make(reqURL, dynamicValues) if err == io.EOF { break } @@ -245,7 +241,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou dynamicValues = generators.MergeMaps(dynamicValues, event.OperatorsResult.DynamicValues) } if hasInteractMarkers && request.options.Interactsh != nil { - request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ + request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, &interactsh.RequestData{ MakeResultFunc: request.MakeResultEvent, Event: event, Operators: request.CompiledOperators, diff --git a/v2/pkg/protocols/http/request_generator.go b/v2/pkg/protocols/http/request_generator.go index 90c5d0fb..78337583 100644 --- a/v2/pkg/protocols/http/request_generator.go +++ b/v2/pkg/protocols/http/request_generator.go @@ -16,6 +16,7 @@ type requestGenerator struct { request *Request options *protocols.ExecuterOptions payloadIterator *generators.Iterator + interactshURLs []string } // newGenerator creates a new request generator instance diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 76135085..7903ce4f 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -120,11 +120,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input defer conn.Close() _ = conn.SetReadDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second)) - hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators) - var interactURL string - if request.options.Interactsh != nil && hasInteractMarkers { - interactURL = request.options.Interactsh.URL() - } + var interactshURLs []string responseBuilder := &strings.Builder{} reqBuilder := &strings.Builder{} @@ -137,9 +133,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input case "hex": data, err = hex.DecodeString(input.Data) default: - if interactURL != "" { - input.Data = request.options.Interactsh.ReplaceMarkers(input.Data, interactURL) - } + input.Data, interactshURLs = request.options.Interactsh.ReplaceMarkers(input.Data, []string{}) data = []byte(input.Data) } if err != nil { @@ -261,14 +255,14 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input } var event *output.InternalWrappedEvent - if interactURL == "" { + if len(interactshURLs) == 0 { event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { wrappedEvent.OperatorsResult.PayloadValues = payloads }) callback(event) } else if request.options.Interactsh != nil { event = &output.InternalWrappedEvent{InternalEvent: outputEvent} - request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ + request.options.Interactsh.RequestEvent(interactshURLs, &interactsh.RequestData{ MakeResultFunc: request.MakeResultEvent, Event: event, Operators: request.CompiledOperators, From 8ad3ebcd05be6adf03d6d4e11f439a450f4f0a35 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 5 Nov 2021 03:01:41 +0530 Subject: [PATCH 089/146] Made code changes as per review comments --- v2/cmd/functional-test/main.go | 2 +- v2/cmd/integration-test/dns.go | 2 +- v2/cmd/integration-test/http.go | 2 +- v2/cmd/integration-test/integration-test.go | 2 +- v2/cmd/integration-test/loader.go | 5 +- v2/cmd/integration-test/network.go | 2 +- v2/cmd/integration-test/websocket.go | 2 +- v2/cmd/integration-test/workflow.go | 2 +- v2/internal/runner/runner.go | 22 +++---- v2/internal/runner/update_test.go | 2 +- v2/pkg/core/engine.go | 4 +- v2/pkg/core/execute.go | 8 +-- v2/pkg/core/inputs/hybrid/hmap.go | 3 +- v2/pkg/operators/common/dsl/dsl.go | 7 +-- v2/pkg/output/output.go | 35 ----------- v2/pkg/progress/progress.go | 24 ------- .../protocols/common/generators/generators.go | 10 +-- .../protocols/common/generators/validate.go | 2 +- v2/pkg/protocols/dns/dns_test.go | 2 +- v2/pkg/protocols/dns/operators_test.go | 2 +- v2/pkg/protocols/dns/request.go | 6 +- v2/pkg/protocols/dns/request_test.go | 2 +- v2/pkg/protocols/file/file_test.go | 2 +- v2/pkg/protocols/file/find_test.go | 2 +- v2/pkg/protocols/file/operators_test.go | 2 +- v2/pkg/protocols/file/request.go | 2 +- v2/pkg/protocols/file/request_test.go | 2 +- v2/pkg/protocols/headless/request.go | 8 +-- v2/pkg/protocols/http/build_request_test.go | 2 +- v2/pkg/protocols/http/http.go | 2 +- v2/pkg/protocols/http/http_test.go | 2 +- v2/pkg/protocols/http/operators_test.go | 2 +- v2/pkg/protocols/http/request.go | 4 +- v2/pkg/protocols/network/network.go | 2 +- v2/pkg/protocols/network/network_test.go | 2 +- v2/pkg/protocols/network/operators_test.go | 2 +- v2/pkg/protocols/network/request.go | 18 +++--- v2/pkg/protocols/network/request_test.go | 2 +- v2/pkg/protocols/offlinehttp/find_test.go | 2 +- .../protocols/offlinehttp/operators_test.go | 2 +- v2/pkg/protocols/protocols.go | 3 + v2/pkg/protocols/ssl/ssl.go | 46 ++++++++++---- v2/pkg/protocols/ssl/ssl_test.go | 2 +- v2/pkg/protocols/websocket/websocket.go | 61 +++++++++--------- v2/pkg/templates/templates.go | 4 +- v2/{internal => pkg}/testutils/integration.go | 0 v2/{internal => pkg}/testutils/testutils.go | 62 ++++++++++++++++++- 47 files changed, 203 insertions(+), 183 deletions(-) rename v2/{internal => pkg}/testutils/integration.go (100%) rename v2/{internal => pkg}/testutils/testutils.go (57%) diff --git a/v2/cmd/functional-test/main.go b/v2/cmd/functional-test/main.go index fc96fde8..7f05014c 100644 --- a/v2/cmd/functional-test/main.go +++ b/v2/cmd/functional-test/main.go @@ -11,7 +11,7 @@ import ( "github.com/logrusorgru/aurora" "github.com/pkg/errors" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var ( diff --git a/v2/cmd/integration-test/dns.go b/v2/cmd/integration-test/dns.go index 3e1ae814..5bb2ed2c 100644 --- a/v2/cmd/integration-test/dns.go +++ b/v2/cmd/integration-test/dns.go @@ -1,7 +1,7 @@ package main import ( - "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var dnsTestCases = map[string]testutils.TestCase{ diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index a81111c4..8805fef8 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -11,7 +11,7 @@ import ( "github.com/julienschmidt/httprouter" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var httpTestcases = map[string]testutils.TestCase{ diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index b854e0a4..47005ed1 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/logrusorgru/aurora" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var ( diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go index 8c44bc20..e07c45c2 100644 --- a/v2/cmd/integration-test/loader.go +++ b/v2/cmd/integration-test/loader.go @@ -2,12 +2,13 @@ package main import ( "fmt" - "github.com/julienschmidt/httprouter" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "net/http" "net/http/httptest" "os" "strings" + + "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var loaderTestcases = map[string]testutils.TestCase{ diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go index e170b07b..ac34c5fd 100644 --- a/v2/cmd/integration-test/network.go +++ b/v2/cmd/integration-test/network.go @@ -3,7 +3,7 @@ package main import ( "net" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var networkTestcases = map[string]testutils.TestCase{ diff --git a/v2/cmd/integration-test/websocket.go b/v2/cmd/integration-test/websocket.go index be20ba77..b5b0e5a3 100644 --- a/v2/cmd/integration-test/websocket.go +++ b/v2/cmd/integration-test/websocket.go @@ -5,7 +5,7 @@ import ( "strings" "github.com/gobwas/ws/wsutil" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var websocketTestCases = map[string]testutils.TestCase{ diff --git a/v2/cmd/integration-test/workflow.go b/v2/cmd/integration-test/workflow.go index 5f39b4eb..31202d09 100644 --- a/v2/cmd/integration-test/workflow.go +++ b/v2/cmd/integration-test/workflow.go @@ -7,7 +7,7 @@ import ( "github.com/julienschmidt/httprouter" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var workflowTestcases = map[string]testutils.TestCase{ diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index fb57b32b..dec4d16b 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -145,18 +145,16 @@ func New(options *types.Options) (*Runner, error) { } if !options.NoInteractsh { - interactshClient, err := interactsh.New(&interactsh.Options{ - ServerURL: options.InteractshURL, - Authorization: options.InteractshToken, - CacheSize: int64(options.InteractionsCacheSize), - Eviction: time.Duration(options.InteractionsEviction) * time.Second, - ColldownPeriod: time.Duration(options.InteractionsColldownPeriod) * time.Second, - PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second, - Output: runner.output, - IssuesClient: runner.issuesClient, - Progress: runner.progress, - Debug: runner.options.Debug, - }) + opts := interactsh.NewDefaultOptions(runner.output, runner.issuesClient, runner.progress) + opts.Debug = runner.options.Debug + opts.ServerURL = options.InteractshURL + opts.Authorization = options.InteractshToken + opts.CacheSize = int64(options.InteractionsCacheSize) + opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second + opts.ColldownPeriod = time.Duration(options.InteractionsColldownPeriod) * time.Second + opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second + + interactshClient, err := interactsh.New(opts) if err != nil { gologger.Error().Msgf("Could not create interactsh client: %s", err) } else { diff --git a/v2/internal/runner/update_test.go b/v2/internal/runner/update_test.go index dc57140c..bc088834 100644 --- a/v2/internal/runner/update_test.go +++ b/v2/internal/runner/update_test.go @@ -13,8 +13,8 @@ import ( "testing" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/stretchr/testify/require" ) diff --git a/v2/pkg/core/engine.go b/v2/pkg/core/engine.go index 5d7cf946..4ea9b541 100644 --- a/v2/pkg/core/engine.go +++ b/v2/pkg/core/engine.go @@ -27,8 +27,8 @@ type Engine struct { type InputProvider interface { // Count returns the number of items for input provider Count() int64 - // Scan iterates the input and for each found item calls the callback - // until the input provider has been exhausted. + // Scan iterates the input and each found item is passed to the + // callback consumer. Scan(callback func(value string)) } diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index ee312b0c..4408d289 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -17,12 +17,12 @@ import ( // // All the execution logic for the templates/workflows happens in this part // of the engine. -func (e *Engine) Execute(templates []*templates.Template, input InputProvider) *atomic.Bool { - return e.ExecuteWithOpts(templates, input, false) +func (e *Engine) Execute(templates []*templates.Template, target InputProvider) *atomic.Bool { + return e.ExecuteWithOpts(templates, target, false) } // ExecuteWithOpts is execute with the full options -func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, input InputProvider, noCluster bool) *atomic.Bool { +func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target InputProvider, noCluster bool) *atomic.Bool { var finalTemplates []*templates.Template if !noCluster { finalTemplates, _ = e.ClusterTemplates(templatesList) @@ -48,7 +48,7 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, input Inpu e.executeSelfContainedTemplateWithInput(template, results) default: // All other request types are executed here - e.executeModelWithInput(templateType, template, input, results) + e.executeModelWithInput(templateType, template, target, results) } wg.Done() } diff --git a/v2/pkg/core/inputs/hybrid/hmap.go b/v2/pkg/core/inputs/hybrid/hmap.go index 1eb4f5e4..406331fe 100644 --- a/v2/pkg/core/inputs/hybrid/hmap.go +++ b/v2/pkg/core/inputs/hybrid/hmap.go @@ -119,7 +119,8 @@ func (i *Input) Count() int64 { return i.inputCount } -// Scan calls an input provider till the callback is exhausted +// Scan iterates the input and each found item is passed to the +// callback consumer. func (i *Input) Scan(callback func(value string)) { callbackFunc := func(k, _ []byte) error { callback(string(k)) diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 4e7ffa2c..cfeede8f 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -332,7 +332,7 @@ var functions = map[string]govaluate.ExpressionFunction{ } now := time.Now() offset := now.Add(time.Duration(seconds) * time.Second) - return offset.Unix(), nil + return float64(offset.Unix()), nil }, // Time Functions "waitfor": func(args ...interface{}) (interface{}, error) { @@ -363,11 +363,6 @@ var functions = map[string]govaluate.ExpressionFunction{ gologger.Info().Msgf("print_debug value: %s", fmt.Sprint(args)) return true, nil }, - // is_before_now compares a timestamp and returns true if the first - // passed argument is a time.Time that has already passed. - "time_now": func(args ...interface{}) (interface{}, error) { - return float64(time.Now().Unix()), nil - }, } // HelperFunctions returns the dsl helper functions diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 96850cb0..991c9923 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -224,38 +224,3 @@ func (w *StandardWriter) Close() { w.errorFile.Close() } } - -// MockOutputWriter is a mocked output writer. -type MockOutputWriter struct { - aurora aurora.Aurora - RequestCallback func(templateID, url, requestType string, err error) - WriteCallback func(o *ResultEvent) -} - -// NewMockOutputWriter creates a new mock output writer -func NewMockOutputWriter() *MockOutputWriter { - return &MockOutputWriter{aurora: aurora.NewAurora(false)} -} - -// Close closes the output writer interface -func (m *MockOutputWriter) Close() {} - -// Colorizer returns the colorizer instance for writer -func (m *MockOutputWriter) Colorizer() aurora.Aurora { - return m.aurora -} - -// Write writes the event to file and/or screen. -func (m *MockOutputWriter) Write(result *ResultEvent) error { - if m.WriteCallback != nil { - m.WriteCallback(result) - } - return nil -} - -// Request writes a log the requests trace log -func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) { - if m.RequestCallback != nil { - m.RequestCallback(templateID, url, requestType, err) - } -} diff --git a/v2/pkg/progress/progress.go b/v2/pkg/progress/progress.go index fcd34340..220ff49a 100644 --- a/v2/pkg/progress/progress.go +++ b/v2/pkg/progress/progress.go @@ -247,27 +247,3 @@ func (p *StatsTicker) Stop() { _ = p.server.Shutdown(context.Background()) } } - -type MockProgressClient struct{} - -// Stop stops the progress recorder. -func (m *MockProgressClient) Stop() {} - -// Init inits the progress bar with initial details for scan -func (m *MockProgressClient) Init(hostCount int64, rulesCount int, requestCount int64) {} - -// AddToTotal adds a value to the total request count -func (m *MockProgressClient) AddToTotal(delta int64) {} - -// IncrementRequests increments the requests counter by 1. -func (m *MockProgressClient) IncrementRequests() {} - -// IncrementMatched increments the matched counter by 1. -func (m *MockProgressClient) IncrementMatched() {} - -// IncrementErrorsBy increments the error counter by count. -func (m *MockProgressClient) IncrementErrorsBy(count int64) {} - -// IncrementFailedRequestsBy increments the number of requests counter by count -// along with errors. -func (m *MockProgressClient) IncrementFailedRequestsBy(count int64) {} diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 47b2fa44..f3aaa8a3 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -7,14 +7,14 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog" ) -// Generator is the generator struct for generating payloads -type Generator struct { +// PayloadGenerator is the generator struct for generating payloads +type PayloadGenerator struct { Type AttackType payloads map[string][]string } // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog *catalog.Catalog) (*Generator, error) { +func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog *catalog.Catalog) (*PayloadGenerator, error) { if attackType.String() == "" { attackType = BatteringRamAttack } @@ -35,7 +35,7 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st } } - generator := &Generator{} + generator := &PayloadGenerator{} if err := generator.validate(payloads, templatePath); err != nil { return nil, err } @@ -66,7 +66,7 @@ type Iterator struct { } // NewIterator creates a new iterator for the payloads generator -func (g *Generator) NewIterator() *Iterator { +func (g *PayloadGenerator) NewIterator() *Iterator { var payloads []*payloadIterator for name, values := range g.payloads { diff --git a/v2/pkg/protocols/common/generators/validate.go b/v2/pkg/protocols/common/generators/validate.go index 0384f890..294ddca3 100644 --- a/v2/pkg/protocols/common/generators/validate.go +++ b/v2/pkg/protocols/common/generators/validate.go @@ -11,7 +11,7 @@ import ( ) // validate validates the payloads if any. -func (g *Generator) validate(payloads map[string]interface{}, templatePath string) error { +func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePath string) error { for name, payload := range payloads { switch pt := payload.(type) { case string: diff --git a/v2/pkg/protocols/dns/dns_test.go b/v2/pkg/protocols/dns/dns_test.go index d01f84bd..b0f8dddf 100644 --- a/v2/pkg/protocols/dns/dns_test.go +++ b/v2/pkg/protocols/dns/dns_test.go @@ -5,9 +5,9 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestGenerateDNSVariables(t *testing.T) { diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index 0e4c6ff3..e5618f61 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -8,13 +8,13 @@ import ( "github.com/miekg/dns" "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestResponseToDSLMap(t *testing.T) { diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 05d4930d..2c916567 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -35,7 +35,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review // Compile each request for the template based on the URL compiledRequest, err := request.Make(domain) if err != nil { - request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) + request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not build request") } @@ -53,7 +53,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review // Send the request to the target servers response, err := request.dnsClient.Do(compiledRequest) if err != nil { - request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) + request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) } if response == nil { @@ -61,7 +61,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } request.options.Progress.IncrementRequests() - request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) + request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err) gologger.Verbose().Msgf("[%s] Sent DNS request to %s\n", request.options.TemplateID, domain) outputEvent := request.responseToDSLMap(compiledRequest, response, input, input) diff --git a/v2/pkg/protocols/dns/request_test.go b/v2/pkg/protocols/dns/request_test.go index bc5f230b..d2ed8399 100644 --- a/v2/pkg/protocols/dns/request_test.go +++ b/v2/pkg/protocols/dns/request_test.go @@ -5,13 +5,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestDNSExecuteWithResults(t *testing.T) { diff --git a/v2/pkg/protocols/file/file_test.go b/v2/pkg/protocols/file/file_test.go index 7338e146..d568f981 100644 --- a/v2/pkg/protocols/file/file_test.go +++ b/v2/pkg/protocols/file/file_test.go @@ -5,9 +5,9 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestFileCompile(t *testing.T) { diff --git a/v2/pkg/protocols/file/find_test.go b/v2/pkg/protocols/file/find_test.go index a2e2e9f3..7262bb26 100644 --- a/v2/pkg/protocols/file/find_test.go +++ b/v2/pkg/protocols/file/find_test.go @@ -8,9 +8,9 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestFindInputPaths(t *testing.T) { diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index da40c76d..f85ff6c8 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -5,13 +5,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestResponseToDSLMap(t *testing.T) { diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index ef650bab..18e15655 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -74,7 +74,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review }) wg.Wait() if err != nil { - request.options.Output.Request(request.options.TemplatePath, input, "file", err) + request.options.Output.Request(request.options.TemplatePath, input, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } diff --git a/v2/pkg/protocols/file/request_test.go b/v2/pkg/protocols/file/request_test.go index 0c9f7e9c..bccfae72 100644 --- a/v2/pkg/protocols/file/request_test.go +++ b/v2/pkg/protocols/file/request_test.go @@ -8,13 +8,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestFileExecuteWithResults(t *testing.T) { diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index dc1d3467..89de2eaa 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -26,7 +26,7 @@ func (request *Request) Type() templateTypes.ProtocolType { func (request *Request) ExecuteWithResults(inputURL string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { instance, err := request.options.Browser.NewInstance() if err != nil { - request.options.Output.Request(request.options.TemplatePath, inputURL, "headless", err) + request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } @@ -34,19 +34,19 @@ func (request *Request) ExecuteWithResults(inputURL string, metadata, previous o parsedURL, err := url.Parse(inputURL) if err != nil { - request.options.Output.Request(request.options.TemplatePath, inputURL, "headless", err) + request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } out, page, err := instance.Run(parsedURL, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second) if err != nil { - request.options.Output.Request(request.options.TemplatePath, inputURL, "headless", err) + request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } defer page.Close() - request.options.Output.Request(request.options.TemplatePath, inputURL, "headless", nil) + request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), nil) request.options.Progress.IncrementRequests() gologger.Verbose().Msgf("Sent Headless request to %s", inputURL) diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index c5cc3aaf..19446393 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -6,10 +6,10 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestBaseURLWithTemplatePrefs(t *testing.T) { diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index df79e8f4..f85e49c2 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -131,7 +131,7 @@ type Request struct { options *protocols.ExecuterOptions totalRequests int customHeaders map[string]string - generator *generators.Generator // optional, only enabled when using payloads + generator *generators.PayloadGenerator // optional, only enabled when using payloads httpClient *retryablehttp.Client rawhttpClient *rawhttp.Client dynamicValues map[string]interface{} diff --git a/v2/pkg/protocols/http/http_test.go b/v2/pkg/protocols/http/http_test.go index 69a71fb2..4da40ce6 100644 --- a/v2/pkg/protocols/http/http_test.go +++ b/v2/pkg/protocols/http/http_test.go @@ -5,10 +5,10 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestHTTPCompile(t *testing.T) { diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index e2ecbdf5..bc7b888a 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -7,13 +7,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestResponseToDSLMap(t *testing.T) { diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index af5075f3..3a455472 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -363,7 +363,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate _, _ = io.CopyN(ioutil.Discard, resp.Body, drainReqSize) resp.Body.Close() } - request.options.Output.Request(request.options.TemplatePath, formedURL, "http", err) + request.options.Output.Request(request.options.TemplatePath, formedURL, request.Type().String(), err) request.options.Progress.IncrementErrorsBy(1) // If we have interactsh markers and request times out, still send @@ -401,7 +401,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", request.options.TemplateID, formedURL) - request.options.Output.Request(request.options.TemplatePath, formedURL, "http", err) + request.options.Output.Request(request.options.TemplatePath, formedURL, request.Type().String(), err) duration := time.Since(timeStart) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 6b3cbe2a..83c9ac81 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -75,7 +75,7 @@ type Request struct { operators.Operators `yaml:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-"` - generator *generators.Generator + generator *generators.PayloadGenerator // cache any variables that may be needed for operation. dialer *fastdialer.Dialer options *protocols.ExecuterOptions diff --git a/v2/pkg/protocols/network/network_test.go b/v2/pkg/protocols/network/network_test.go index 180b3419..26b83502 100644 --- a/v2/pkg/protocols/network/network_test.go +++ b/v2/pkg/protocols/network/network_test.go @@ -5,9 +5,9 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestNetworkCompileMake(t *testing.T) { diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index 735d9af3..9e8a193c 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -5,13 +5,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestResponseToDSLMap(t *testing.T) { diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index ed52117f..14b563f3 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -42,7 +42,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review address, err = getAddress(input) } if err != nil { - request.options.Output.Request(request.options.TemplatePath, input, "network", err) + request.options.Output.Request(request.options.TemplatePath, input, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not get address from url") } @@ -71,7 +71,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review func (request *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { if !strings.Contains(actualAddress, ":") { err := errors.New("no port provided in network protocol request") - request.options.Output.Request(request.options.TemplatePath, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return err } @@ -119,7 +119,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input conn, err = request.dialer.Dial(context.Background(), "tcp", actualAddress) } if err != nil { - request.options.Output.Request(request.options.TemplatePath, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server request") } @@ -149,7 +149,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input data = []byte(input.Data) } if err != nil { - request.options.Output.Request(request.options.TemplatePath, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } @@ -157,7 +157,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input finalData, dataErr := expressions.EvaluateByte(data, payloads) if dataErr != nil { - request.options.Output.Request(request.options.TemplatePath, address, "network", dataErr) + request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), dataErr) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } @@ -168,7 +168,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input return nil } if _, err := conn.Write(finalData); err != nil { - request.options.Output.Request(request.options.TemplatePath, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } @@ -203,7 +203,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input } } - request.options.Output.Request(request.options.TemplatePath, actualAddress, "network", err) + request.options.Output.Request(request.options.TemplatePath, actualAddress, request.Type().String(), err) gologger.Verbose().Msgf("Sent TCP request to %s", actualAddress) bufferSize := 1024 @@ -234,7 +234,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input buf := make([]byte, bufferSize) nBuf, err := conn.Read(buf) if err != nil && !os.IsTimeout(err) { - request.options.Output.Request(request.options.TemplatePath, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) closeTimer(readInterval) return errors.Wrap(err, "could not read from server") } @@ -247,7 +247,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input final = make([]byte, bufferSize) n, err = conn.Read(final) if err != nil && err != io.EOF { - request.options.Output.Request(request.options.TemplatePath, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) return errors.Wrap(err, "could not read from server") } responseBuilder.Write(final[:n]) diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index d1f266b4..8b44091e 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -10,13 +10,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestNetworkExecuteWithResults(t *testing.T) { diff --git a/v2/pkg/protocols/offlinehttp/find_test.go b/v2/pkg/protocols/offlinehttp/find_test.go index 369ef440..d6b933ce 100644 --- a/v2/pkg/protocols/offlinehttp/find_test.go +++ b/v2/pkg/protocols/offlinehttp/find_test.go @@ -8,10 +8,10 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestFindResponses(t *testing.T) { diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index 2f4b91fd..8669d8b5 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -7,13 +7,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestResponseToDSLMap(t *testing.T) { diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index a686b9c4..51cfab06 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -96,6 +97,8 @@ type Request interface { MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent // GetCompiledOperators returns a list of the compiled operators GetCompiledOperators() []*operators.Operators + // Type returns the type of the protocol request + Type() templateTypes.ProtocolType } // OutputEventCallback is a callback event for any results found during scanning. diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index 8f624d5b..ba26c8e3 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -18,6 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" @@ -30,6 +31,9 @@ type Request struct { // Operators for the current request go here. operators.Operators `yaml:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-"` + // description: | + // Address contains address for the request + Address string `yaml:"address,omitempty" jsonschema:"title=address for the ssl request,description=Address contains address for the request"` // cache any variables that may be needed for operation. dialer *fastdialer.Dialer @@ -72,28 +76,45 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous if err != nil { return nil } - hostname, _, _ := net.SplitHostPort(address) + hostname, port, _ := net.SplitHostPort(address) + requestOptions := request.options + payloadValues := make(map[string]interface{}) + for k, v := range dynamicValues { + payloadValues[k] = v + } + payloadValues["Hostname"] = address + payloadValues["Host"] = hostname + payloadValues["Port"] = port + + finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues) + if dataErr != nil { + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) + requestOptions.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not evaluate template expressions") + } + + addressToDial := string(finalAddress) config := &tls.Config{InsecureSkipVerify: true, ServerName: hostname} - conn, err := request.dialer.DialTLSWithConfig(context.Background(), "tcp", address, config) + conn, err := request.dialer.DialTLSWithConfig(context.Background(), "tcp", addressToDial, config) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "ssl", err) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) + requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server") } defer conn.Close() - _ = conn.SetReadDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second)) + _ = conn.SetReadDeadline(time.Now().Add(time.Duration(requestOptions.Options.Timeout) * time.Second)) connTLS, ok := conn.(*tls.Conn) if !ok { return nil } - request.options.Output.Request(request.options.TemplateID, address, "ssl", err) + requestOptions.Output.Request(requestOptions.TemplateID, address, request.Type().String(), err) gologger.Verbose().Msgf("Sent SSL request to %s", address) - if request.options.Options.Debug || request.options.Options.DebugRequests { - gologger.Debug().Str("address", input).Msgf("[%s] Dumped SSL request for %s", request.options.TemplateID, input) + if requestOptions.Options.Debug || requestOptions.Options.DebugRequests { + gologger.Debug().Str("address", input).Msgf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input) } state := connTLS.ConnectionState() @@ -110,13 +131,14 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous data["response"] = jsonDataString data["host"] = input + data["matched"] = addressToDial data["not_after"] = float64(cert.NotAfter.Unix()) data["ip"] = request.dialer.GetDialedIP(hostname) - event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse) - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped SSL response for %s", request.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, request.options.Options.NoColor, false)) + event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse) + if requestOptions.Options.Debug || requestOptions.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, requestOptions.Options.NoColor, false)) } callback(event) return nil diff --git a/v2/pkg/protocols/ssl/ssl_test.go b/v2/pkg/protocols/ssl/ssl_test.go index fef44fbb..10522d6a 100644 --- a/v2/pkg/protocols/ssl/ssl_test.go +++ b/v2/pkg/protocols/ssl/ssl_test.go @@ -3,10 +3,10 @@ package ssl import ( "testing" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/stretchr/testify/require" ) diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 7731dc29..7d2aca24 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -65,7 +65,7 @@ type Request struct { // be provided as payload which will be read on run-time. Payloads map[string]interface{} `yaml:"payloads,omitempty" jsonschema:"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request"` - generator *generators.Generator + generator *generators.PayloadGenerator // cache any variables that may be needed for operation. dialer *fastdialer.Dialer @@ -174,46 +174,44 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam payloadValues["Scheme"] = parsed.Scheme payloadValues["Path"] = parsed.Path + requestOptions := request.options for key, value := range request.Headers { finalData, dataErr := expressions.EvaluateByte([]byte(value), payloadValues) if dataErr != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) + requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } header.Set(key, string(finalData)) } websocketDialer := ws.Dialer{ Header: ws.HandshakeHeaderHTTP(header), - Timeout: time.Duration(request.options.Options.Timeout) * time.Second, + Timeout: time.Duration(requestOptions.Options.Timeout) * time.Second, NetDial: request.dialer.Dial, TLSConfig: &tls.Config{InsecureSkipVerify: true, ServerName: hostname}, } finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues) if dataErr != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) + requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } addressToDial := string(finalAddress) parsedAddress, err := url.Parse(addressToDial) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) + requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not parse input url") } - - if parsed.Path != "" && parsed.Path != "/" { - parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path) - addressToDial = parsedAddress.String() - } + parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path) + addressToDial = parsedAddress.String() conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", err) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) + requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server") } defer conn.Close() @@ -225,18 +223,18 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam events, requestOutput, err := request.readWriteInputWebsocket(conn, payloadValues, input, responseBuilder) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", err) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) + requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not read write response") } - request.options.Progress.IncrementRequests() + requestOptions.Progress.IncrementRequests() - if request.options.Options.Debug || request.options.Options.DebugRequests { - gologger.Debug().Str("address", input).Msgf("[%s] Dumped Websocket request for %s", request.options.TemplateID, input) + if requestOptions.Options.Debug || requestOptions.Options.DebugRequests { + gologger.Debug().Str("address", input).Msgf("[%s] Dumped Websocket request for %s", requestOptions.TemplateID, input) gologger.Print().Msgf("%s", requestOutput) } - request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) gologger.Verbose().Msgf("Sent Websocket request to %s", input) data := make(map[string]interface{}) @@ -253,13 +251,13 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam data["matched"] = addressToDial data["ip"] = request.dialer.GetDialedIP(hostname) - event := eventcreator.CreateEventWithAdditionalOptions(request, data, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + event := eventcreator.CreateEventWithAdditionalOptions(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { internalWrappedEvent.OperatorsResult.PayloadValues = payloadValues }) - if request.options.Options.Debug || request.options.Options.DebugResponse { + if requestOptions.Options.Debug || requestOptions.Options.DebugResponse { responseOutput := responseBuilder.String() - gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", request.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, request.options.Options.NoColor, false)) + gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", requestOptions.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, requestOptions.Options.NoColor, false)) } callback(event) @@ -270,28 +268,29 @@ func (request *Request) readWriteInputWebsocket(conn net.Conn, payloadValues map reqBuilder := &strings.Builder{} inputEvents := make(map[string]interface{}) + requestOptions := request.options for _, req := range request.Inputs { reqBuilder.Grow(len(req.Data)) finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues) if dataErr != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) + requestOptions.Progress.IncrementFailedRequestsBy(1) return nil, "", errors.Wrap(dataErr, "could not evaluate template expressions") } reqBuilder.WriteString(string(finalData)) err = wsutil.WriteClientMessage(conn, ws.OpText, finalData) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", err) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) + requestOptions.Progress.IncrementFailedRequestsBy(1) return nil, "", errors.Wrap(err, "could not write request to server") } msg, opCode, err := wsutil.ReadServerData(conn) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", err) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) + requestOptions.Progress.IncrementFailedRequestsBy(1) return nil, "", errors.Wrap(err, "could not write request to server") } // Only perform matching and writes in case we recieve diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index d44ac8e7..39e0e447 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -83,8 +83,8 @@ type Template struct { Path string `yaml:"-" json:"-"` } -// TemplateTypes is a list of accepted template types -var TemplateTypes = []string{ +// TemplateProtocols is a list of accepted template protocols +var TemplateProtocols = []string{ "dns", "file", "http", diff --git a/v2/internal/testutils/integration.go b/v2/pkg/testutils/integration.go similarity index 100% rename from v2/internal/testutils/integration.go rename to v2/pkg/testutils/integration.go diff --git a/v2/internal/testutils/testutils.go b/v2/pkg/testutils/testutils.go similarity index 57% rename from v2/internal/testutils/testutils.go rename to v2/pkg/testutils/testutils.go index e58c7079..0ea388f7 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -3,6 +3,7 @@ package testutils import ( "go.uber.org/ratelimit" + "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/model" @@ -73,7 +74,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco TemplateID: info.ID, TemplateInfo: info.Info, TemplatePath: info.Path, - Output: output.NewMockOutputWriter(), + Output: NewMockOutputWriter(), Options: options, Progress: progressImpl, ProjectFile: nil, @@ -90,3 +91,62 @@ type NoopWriter struct{} // Write writes the data to an output writer. func (n *NoopWriter) Write(data []byte, level levels.Level) {} + +// MockOutputWriter is a mocked output writer. +type MockOutputWriter struct { + aurora aurora.Aurora + RequestCallback func(templateID, url, requestType string, err error) + WriteCallback func(o *output.ResultEvent) +} + +// NewMockOutputWriter creates a new mock output writer +func NewMockOutputWriter() *MockOutputWriter { + return &MockOutputWriter{aurora: aurora.NewAurora(false)} +} + +// Close closes the output writer interface +func (m *MockOutputWriter) Close() {} + +// Colorizer returns the colorizer instance for writer +func (m *MockOutputWriter) Colorizer() aurora.Aurora { + return m.aurora +} + +// Write writes the event to file and/or screen. +func (m *MockOutputWriter) Write(result *output.ResultEvent) error { + if m.WriteCallback != nil { + m.WriteCallback(result) + } + return nil +} + +// Request writes a log the requests trace log +func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) { + if m.RequestCallback != nil { + m.RequestCallback(templateID, url, requestType, err) + } +} + +type MockProgressClient struct{} + +// Stop stops the progress recorder. +func (m *MockProgressClient) Stop() {} + +// Init inits the progress bar with initial details for scan +func (m *MockProgressClient) Init(hostCount int64, rulesCount int, requestCount int64) {} + +// AddToTotal adds a value to the total request count +func (m *MockProgressClient) AddToTotal(delta int64) {} + +// IncrementRequests increments the requests counter by 1. +func (m *MockProgressClient) IncrementRequests() {} + +// IncrementMatched increments the matched counter by 1. +func (m *MockProgressClient) IncrementMatched() {} + +// IncrementErrorsBy increments the error counter by count. +func (m *MockProgressClient) IncrementErrorsBy(count int64) {} + +// IncrementFailedRequestsBy increments the number of requests counter by count +// along with errors. +func (m *MockProgressClient) IncrementFailedRequestsBy(count int64) {} From 4d8eaad0a3050a308f308e168b85b6537e5bab56 Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Fri, 5 Nov 2021 15:27:49 +0530 Subject: [PATCH 090/146] add unit test for unique interactsh url #1068 --- v2/cmd/nuclei/main.go | 2 +- v2/internal/runner/runner.go | 2 +- v2/internal/testutils/testutils.go | 77 ++++++++++--------- .../protocols/common/interactsh/interactsh.go | 6 +- v2/pkg/protocols/http/build_request_test.go | 44 ++++++++++- v2/pkg/types/types.go | 4 +- 6 files changed, 88 insertions(+), 47 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 7f43868f..16997964 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -106,7 +106,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"), flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"), flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"), - flagSet.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"), + flagSet.IntVar(&options.InteractionsCooldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"), flagSet.BoolVarP(&options.NoInteractsh, "no-interactsh", "ni", false, "disable interactsh server for OAST testing, exclude OAST based templates"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index de1caff9..c9a0ab84 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -249,7 +249,7 @@ func New(options *types.Options) (*Runner, error) { Authorization: options.InteractshToken, CacheSize: int64(options.InteractionsCacheSize), Eviction: time.Duration(options.InteractionsEviction) * time.Second, - ColldownPeriod: time.Duration(options.InteractionsColldownPeriod) * time.Second, + ColldownPeriod: time.Duration(options.InteractionsCooldownPeriod) * time.Second, PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second, Output: runner.output, IssuesClient: runner.issuesClient, diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index d62b1d63..79e42642 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -22,42 +22,47 @@ func Init(options *types.Options) { // DefaultOptions is the default options structure for nuclei during mocking. var DefaultOptions = &types.Options{ - Metrics: false, - Debug: false, - DebugRequests: false, - DebugResponse: false, - Silent: false, - Version: false, - Verbose: false, - NoColor: true, - UpdateTemplates: false, - JSON: false, - JSONRequests: false, - EnableProgressBar: false, - TemplatesVersion: false, - TemplateList: false, - Stdin: false, - StopAtFirstMatch: false, - NoMeta: false, - Project: false, - MetricsPort: 0, - BulkSize: 25, - TemplateThreads: 10, - Timeout: 5, - Retries: 1, - RateLimit: 150, - ProjectPath: "", - Severities: severity.Severities{}, - Targets: []string{}, - TargetsFilePath: "", - Output: "", - ProxyURL: "", - ProxySocksURL: "", - TemplatesDirectory: "", - TraceLogFile: "", - Templates: []string{}, - ExcludedTemplates: []string{}, - CustomHeaders: []string{}, + Metrics: false, + Debug: false, + DebugRequests: false, + DebugResponse: false, + Silent: false, + Version: false, + Verbose: false, + NoColor: true, + UpdateTemplates: false, + JSON: false, + JSONRequests: false, + EnableProgressBar: false, + TemplatesVersion: false, + TemplateList: false, + Stdin: false, + StopAtFirstMatch: false, + NoMeta: false, + Project: false, + MetricsPort: 0, + BulkSize: 25, + TemplateThreads: 10, + Timeout: 5, + Retries: 1, + RateLimit: 150, + ProjectPath: "", + Severities: severity.Severities{}, + Targets: []string{}, + TargetsFilePath: "", + Output: "", + ProxyURL: "", + ProxySocksURL: "", + TemplatesDirectory: "", + TraceLogFile: "", + Templates: []string{}, + ExcludedTemplates: []string{}, + CustomHeaders: []string{}, + InteractshURL: "https://interactsh.com", + InteractionsCacheSize: 5000, + InteractionsEviction: 60, + InteractionsCooldownPeriod: 5, + InteractionsPollDuration: 5, } // MockOutputWriter is a mocked output writer. diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 5fff7ac5..2fc78857 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -241,16 +241,12 @@ func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) { c.requests.Set(id, data, c.eviction) return } - matched := false for _, interaction := range interactions { if c.processInteractionForRequest(interaction, data) { - matched = true + c.interactions.Delete(id) break } } - if matched { - c.interactions.Delete(id) - } } else { c.requests.Set(id, data, c.eviction) } diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index 122d81bc..663fe109 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -3,12 +3,13 @@ package http import ( "net/url" "testing" - - "github.com/stretchr/testify/require" + "time" "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" + "github.com/stretchr/testify/require" ) func TestBaseURLWithTemplatePrefs(t *testing.T) { @@ -198,3 +199,42 @@ Accept-Encoding: gzip`}, authorization = req.request.Header.Get("Authorization") require.Equal(t, "Basic YWRtaW46Z3Vlc3Q=", authorization, "could not get correct authorization headers from raw") } + +func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { + + options := testutils.DefaultOptions + + testutils.Init(options) + templateID := "testing-unique-interactsh" + request := &Request{ + ID: templateID, + Name: "testing", + Path: []string{"{{BaseURL}}/?u=http://{{interactsh-url}}/&href=http://{{interactsh-url}}/&action=http://{{interactsh-url}}/&host={{interactsh-url}}"}, + Method: "GET", + } + executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + ID: templateID, + Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, + }) + err := request.Compile(executerOpts) + require.Nil(t, err, "could not compile http request") + + generator := request.newGenerator() + + generator.options.Interactsh, err = interactsh.New(&interactsh.Options{ + ServerURL: options.InteractshURL, + CacheSize: int64(options.InteractionsCacheSize), + Eviction: time.Duration(options.InteractionsEviction) * time.Second, + ColldownPeriod: time.Duration(options.InteractionsCooldownPeriod) * time.Second, + PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second, + }) + require.Nil(t, err, "could not create interactsh client") + + got, err := generator.Make("https://example.com", map[string]interface{}{}) + require.Nil(t, err, "could not make http request") + + //check if all the interactsh markers are replaced with unique urls + require.NotContains(t, got.request.URL.String(), "{{interactsh-url}}", "could not get correct interactsh url") + //check the length of returned urls + require.Equal(t, len(got.interactshURLs), 4, "could not get correct interactsh url") +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index c19ad18a..50a8c78a 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -100,9 +100,9 @@ type Options struct { // Eviction is the number of seconds after which to automatically discard // interaction requests. InteractionsEviction int - // InteractionsColldownPeriod is additional seconds to wait for interactions after closing + // InteractionsCooldownPeriod is additional seconds to wait for interactions after closing // of the poller. - InteractionsColldownPeriod int + InteractionsCooldownPeriod int // OfflineHTTP is a flag that specific offline processing of http response // using same matchers/extractors from http protocol without the need // to send a new request, reading responses from a file. From 0a09b9e451ddbdb4e047ed575aa563ececdb92b6 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 5 Nov 2021 16:59:24 +0530 Subject: [PATCH 091/146] Added test + misc --- v2/pkg/protocols/ssl/ssl_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/ssl/ssl_test.go b/v2/pkg/protocols/ssl/ssl_test.go index 10522d6a..8d7e0217 100644 --- a/v2/pkg/protocols/ssl/ssl_test.go +++ b/v2/pkg/protocols/ssl/ssl_test.go @@ -15,7 +15,9 @@ func TestSSLProtocol(t *testing.T) { testutils.Init(options) templateID := "testing-ssl" - request := &Request{} + request := &Request{ + Address: "{{Hostname}}", + } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, @@ -23,8 +25,12 @@ func TestSSLProtocol(t *testing.T) { err := request.Compile(executerOpts) require.Nil(t, err, "could not compile ssl request") - err = request.ExecuteWithResults("google.com:443", nil, nil, func(event *output.InternalWrappedEvent) {}) + var gotEvent output.InternalEvent + err = request.ExecuteWithResults("google.com:443", nil, nil, func(event *output.InternalWrappedEvent) { + gotEvent = event.InternalEvent + }) require.Nil(t, err, "could not run ssl request") + require.NotEmpty(t, gotEvent, "could not get event items") } func TestGetAddress(t *testing.T) { From 1c360e0344e1ca2c7106f4519602f1e59925b42d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 5 Nov 2021 17:04:42 +0530 Subject: [PATCH 092/146] Fixed DSL test --- v2/pkg/operators/common/dsl/dsl_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/operators/common/dsl/dsl_test.go b/v2/pkg/operators/common/dsl/dsl_test.go index 42bf5810..3db52977 100644 --- a/v2/pkg/operators/common/dsl/dsl_test.go +++ b/v2/pkg/operators/common/dsl/dsl_test.go @@ -21,7 +21,7 @@ func TestDSLURLEncodeDecode(t *testing.T) { } func TestDSLTimeComparison(t *testing.T) { - compiled, err := govaluate.NewEvaluableExpressionWithFunctions("time_now() > not_after", HelperFunctions()) + compiled, err := govaluate.NewEvaluableExpressionWithFunctions("unixtime() > not_after", HelperFunctions()) require.Nil(t, err, "could not compare time") result, err := compiled.Evaluate(map[string]interface{}{"not_after": float64(time.Now().Unix() - 1000)}) From e8197f127f1a29ba46287a59fa660628e2e0d3b9 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 5 Nov 2021 17:24:23 +0530 Subject: [PATCH 093/146] Fixed rmeote url loader test cases --- v2/pkg/catalog/loader/loader.go | 7 +++++-- v2/pkg/catalog/loader/remote_loader.go | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index a2d7d65f..1a61cb25 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -56,6 +56,8 @@ func NewConfig(options *types.Options, catalog *catalog.Catalog, executerOpts pr loaderConfig := Config{ Templates: options.Templates, Workflows: options.Workflows, + TemplateURLs: options.TemplateURLs, + WorkflowURLs: options.WorkflowURLs, ExcludeTemplates: options.ExcludedTemplates, Tags: options.Tags, ExcludeTags: options.ExcludeTags, @@ -96,7 +98,8 @@ func New(config *Config) (*Store, error) { finalWorkflows: config.Workflows, } - if len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 { + urlbasedTemplatesProvided := len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 + if urlbasedTemplatesProvided { remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs) if err != nil { return store, err @@ -106,7 +109,7 @@ func New(config *Config) (*Store, error) { } // Handle a case with no templates or workflows, where we use base directory - if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 { + if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 && !urlbasedTemplatesProvided { store.finalTemplates = []string{config.TemplatesDirectory} } diff --git a/v2/pkg/catalog/loader/remote_loader.go b/v2/pkg/catalog/loader/remote_loader.go index d8f71eda..c787e960 100644 --- a/v2/pkg/catalog/loader/remote_loader.go +++ b/v2/pkg/catalog/loader/remote_loader.go @@ -3,9 +3,10 @@ package loader import ( "bufio" "fmt" - "github.com/pkg/errors" "net/http" "strings" + + "github.com/pkg/errors" ) type ContentType string From 6d0d3e1c359457bb22ee3afe380c8ebd96b9d4ee Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Fri, 5 Nov 2021 20:00:46 +0530 Subject: [PATCH 094/146] add unique check to test case --- v2/pkg/protocols/http/build_request_test.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index 663fe109..0109498a 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -233,8 +233,22 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { got, err := generator.Make("https://example.com", map[string]interface{}{}) require.Nil(t, err, "could not make http request") - //check if all the interactsh markers are replaced with unique urls + // check if all the interactsh markers are replaced with unique urls require.NotContains(t, got.request.URL.String(), "{{interactsh-url}}", "could not get correct interactsh url") - //check the length of returned urls + // check the length of returned urls require.Equal(t, len(got.interactshURLs), 4, "could not get correct interactsh url") + // check if the interactsh urls are unique + require.True(t, areUnique(got.interactshURLs), "interactsh urls are not unique") +} + +// areUnique checks if the elements of string slice are unique +func areUnique(elements []string) bool { + encountered := map[string]bool{} + for v := range elements { + if encountered[elements[v]] { + return false + } + encountered[elements[v]] = true + } + return true } From af68a30c5c355a58675ab99ca8686250f77595e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Nov 2021 05:04:43 +0000 Subject: [PATCH 095/146] chore(deps): bump golang from 1.17.2-alpine to 1.17.3-alpine Bumps golang from 1.17.2-alpine to 1.17.3-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7d3467d3..c5c46e83 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.2-alpine as build-env +FROM golang:1.17.3-alpine as build-env RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest FROM alpine:3.14 From d772dedef146bf178be399fd163f6006206260d7 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 8 Nov 2021 16:01:45 +0530 Subject: [PATCH 096/146] Make default part for new protocols to response --- v2/pkg/protocols/protocols.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 51cfab06..8333ec86 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -134,7 +134,12 @@ func MakeDefaultResultEvent(request Request, wrapped *output.InternalWrappedEven // MakeDefaultExtractFunc performs extracting operation for an extractor on model and returns true or false. func MakeDefaultExtractFunc(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - item, ok := data[extractor.Part] + part := extractor.Part + if part == "" { + part = "response" + } + + item, ok := data[part] if !ok { return nil } @@ -155,6 +160,11 @@ func MakeDefaultExtractFunc(data map[string]interface{}, extractor *extractors.E // MakeDefaultMatchFunc performs matching operation for a matcher on model and returns true or false. func MakeDefaultMatchFunc(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + part := matcher.Part + if part == "" { + part = "response" + } + partItem, ok := data[matcher.Part] if !ok && len(matcher.DSL) == 0 { return false, nil From ce7534112aa392765a68659642c4ef0ec8c6101d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 8 Nov 2021 16:10:04 +0530 Subject: [PATCH 097/146] Fixed linter issues --- v2/pkg/protocols/protocols.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 8333ec86..7ad9d24a 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -165,7 +165,7 @@ func MakeDefaultMatchFunc(data map[string]interface{}, matcher *matchers.Matcher part = "response" } - partItem, ok := data[matcher.Part] + partItem, ok := data[part] if !ok && len(matcher.DSL) == 0 { return false, nil } From d442c51c1a15e2302773a5d9f740a91eb280b308 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 8 Nov 2021 16:14:47 +0530 Subject: [PATCH 098/146] Changed input to target in core --- v2/pkg/core/execute.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index 4408d289..c9d36cd9 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -66,10 +66,10 @@ func (e *Engine) executeSelfContainedTemplateWithInput(template *templates.Templ } // executeModelWithInput executes a type of template with input -func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template *templates.Template, input InputProvider, results *atomic.Bool) { +func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template *templates.Template, target InputProvider, results *atomic.Bool) { wg := e.workPool.InputPool(templateType) - input.Scan(func(scannedValue string) { + target.Scan(func(scannedValue string) { // Skip if the host has had errors if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue) { return From 88a296c9a7887f4e105d204031221f2dd6d48d4e Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 8 Nov 2021 16:24:47 +0530 Subject: [PATCH 099/146] Adding gzip encoding helpers --- v2/pkg/operators/common/dsl/dsl.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index cfeede8f..378b171c 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -1,6 +1,8 @@ package dsl import ( + "bytes" + "compress/gzip" "crypto/md5" "crypto/sha1" "crypto/sha256" @@ -120,6 +122,16 @@ var functions = map[string]govaluate.ExpressionFunction{ return sEnc, nil }, + "gzip": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } + buffer := &bytes.Buffer{} + if _, err := gzip.NewWriter(buffer).Write([]byte(args[0].(string))); err != nil { + return "", err + } + return buffer.String(), nil + }, // python encodes to base64 with lines of 76 bytes terminated by new line "\n" "base64_py": func(args ...interface{}) (interface{}, error) { if len(args) != 1 { From 7d0529aaf7d972faf9ba1e10748ce9d0f6ca4393 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 8 Nov 2021 17:39:08 +0530 Subject: [PATCH 100/146] Fixed gzip encoding bug + added test case --- v2/pkg/operators/common/dsl/dsl.go | 5 ++++- v2/pkg/operators/common/dsl/dsl_test.go | 17 +++++++++++++++++ .../common/helpers/deserialization/java.go | 8 ++++++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 378b171c..e4cee9c5 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -127,9 +127,12 @@ var functions = map[string]govaluate.ExpressionFunction{ return nil, ErrDSLArguments } buffer := &bytes.Buffer{} - if _, err := gzip.NewWriter(buffer).Write([]byte(args[0].(string))); err != nil { + writer := gzip.NewWriter(buffer) + if _, err := writer.Write([]byte(args[0].(string))); err != nil { return "", err } + _ = writer.Close() + return buffer.String(), nil }, // python encodes to base64 with lines of 76 bytes terminated by new line "\n" diff --git a/v2/pkg/operators/common/dsl/dsl_test.go b/v2/pkg/operators/common/dsl/dsl_test.go index 3db52977..f75baf08 100644 --- a/v2/pkg/operators/common/dsl/dsl_test.go +++ b/v2/pkg/operators/common/dsl/dsl_test.go @@ -1,10 +1,14 @@ package dsl import ( + "compress/gzip" + "io/ioutil" + "strings" "testing" "time" "github.com/Knetic/govaluate" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/stretchr/testify/require" ) @@ -28,3 +32,16 @@ func TestDSLTimeComparison(t *testing.T) { require.Nil(t, err, "could not evaluate compare time") require.Equal(t, true, result, "could not get url encoded data") } + +func TestDSLGzipSerialize(t *testing.T) { + compiled, err := govaluate.NewEvaluableExpressionWithFunctions("gzip(\"hello world\")", HelperFunctions()) + require.Nil(t, err, "could not compare time") + + result, err := compiled.Evaluate(make(map[string]interface{})) + require.Nil(t, err, "could not evaluate compare time") + + reader, _ := gzip.NewReader(strings.NewReader(types.ToString(result))) + data, _ := ioutil.ReadAll(reader) + + require.Equal(t, "hello world", string(data), "could not get gzip encoded data") +} diff --git a/v2/pkg/protocols/common/helpers/deserialization/java.go b/v2/pkg/protocols/common/helpers/deserialization/java.go index ee8ddfb6..0ea107c3 100644 --- a/v2/pkg/protocols/common/helpers/deserialization/java.go +++ b/v2/pkg/protocols/common/helpers/deserialization/java.go @@ -48,15 +48,19 @@ func gadgetEncodingHelper(returnData []byte, encoding string) string { return hex.EncodeToString(returnData) case "gzip": buffer := &bytes.Buffer{} - if _, err := gzip.NewWriter(buffer).Write(returnData); err != nil { + writer := gzip.NewWriter(buffer) + if _, err := writer.Write(returnData); err != nil { return "" } + _ = writer.Close() return buffer.String() case "gzip-base64": buffer := &bytes.Buffer{} - if _, err := gzip.NewWriter(buffer).Write(returnData); err != nil { + writer := gzip.NewWriter(buffer) + if _, err := writer.Write(returnData); err != nil { return "" } + _ = writer.Close() return urlsafeBase64Encode(buffer.Bytes()) case "base64-raw": return base64.StdEncoding.EncodeToString(returnData) From 5e6b6c6b14c5542f8f8c9b2e13e62d5a9aa3ec53 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 8 Nov 2021 17:56:14 +0530 Subject: [PATCH 101/146] Show errors on invalid URL typo --- v2/pkg/protocols/websocket/websocket.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 7d2aca24..cdc40cc0 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -201,9 +201,9 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam addressToDial := string(finalAddress) parsedAddress, err := url.Parse(addressToDial) if err != nil { - requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) requestOptions.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(dataErr, "could not parse input url") + return errors.Wrap(err, "could not parse input url") } parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path) addressToDial = parsedAddress.String() From 9675efa680926500fc2be1c84a65805baf6012d0 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 8 Nov 2021 19:33:54 +0100 Subject: [PATCH 102/146] Adding support for custom headers via CLI in raw http requests --- v2/go.mod | 7 ------- v2/go.sum | 4 ---- v2/pkg/protocols/http/build_request.go | 3 +++ v2/pkg/protocols/http/raw/raw.go | 27 ++++++++++++++++++++++++++ 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 5e6e5f0c..2c003738 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -3,7 +3,6 @@ module github.com/projectdiscovery/nuclei/v2 go 1.17 require ( - github.com/Ice3man543/nvd v1.0.8 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/akrylysov/pogreb v0.10.1 // indirect github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c @@ -20,7 +19,6 @@ require ( github.com/gosuri/uiprogress v0.0.1 // indirect github.com/itchyny/gojq v0.12.4 github.com/json-iterator/go v1.1.12 - github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.16.1 github.com/logrusorgru/aurora v2.0.3+incompatible @@ -49,7 +47,6 @@ require ( github.com/shirou/gopsutil/v3 v3.21.7 github.com/spaolacci/murmur3 v1.1.0 github.com/spf13/cast v1.4.1 - github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.1 @@ -71,16 +68,13 @@ require ( require ( git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect - github.com/PuerkitoBio/goquery v1.6.0 // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect - github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.1.6 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/eggsampler/acme/v3 v3.2.1 // indirect @@ -105,7 +99,6 @@ require ( github.com/microcosm-cc/bluemonday v1.0.15 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect diff --git a/v2/go.sum b/v2/go.sum index a497178f..177689ca 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -42,7 +42,6 @@ github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EF github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= -github.com/Ice3man543/nvd v1.0.8 h1:2CBEgOxyWAkQocnnmEMmRtVPWooPRvcuHFLWj48EM4c= github.com/Ice3man543/nvd v1.0.8/go.mod h1:0DxLJk6revOcJKiZxa2K+rNF/HO1zJO97lqQtXhXfSc= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= @@ -54,7 +53,6 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94= github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -75,7 +73,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= -github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andygrunwald/go-jira v1.14.0 h1:7GT/3qhar2dGJ0kq8w0d63liNyHOnxZsUZ9Pe4+AKBI= github.com/andygrunwald/go-jira v1.14.0/go.mod h1:KMo2f4DgMZA1C9FdImuLc04x4WQhn5derQpnsuBFgqE= @@ -409,7 +406,6 @@ github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0b github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 20f3d480..4b4ac9f2 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -227,6 +227,9 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest // Unsafe option uses rawhttp library if r.request.Unsafe { + if len(r.options.Options.CustomHeaders) > 0 { + _ = rawRequestData.TryFillCustomHeaders(r.options.Options.CustomHeaders) + } unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: generatorValues, original: r.request} return unsafeReq, nil } diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 18bebc39..e2d7f8eb 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -3,6 +3,7 @@ package raw import ( "bufio" "bytes" + "errors" "fmt" "io" "io/ioutil" @@ -146,3 +147,29 @@ func fixUnsafeRequestPath(baseURL *url.URL, requestPath string, request []byte) fixed := bytes.Replace(request, []byte(requestPath), []byte(fixedPath), 1) return fixed } + +// TryFillCustomHeaders after the Host header +func (r *Request) TryFillCustomHeaders(headers []string) error { + unsafeBytes := bytes.ToLower(r.UnsafeRawBytes) + // locate first host header + hostHeaderIndex := bytes.Index(unsafeBytes, []byte("host:")) + if hostHeaderIndex > 0 { + // attempt to locate next newline + newLineIndex := bytes.Index(unsafeBytes[hostHeaderIndex:], []byte("\r\n")) + if newLineIndex > 0 { + newLineIndex += hostHeaderIndex + 2 + // insert custom headers + var buf bytes.Buffer + buf.Write(r.UnsafeRawBytes[:newLineIndex]) + for _, header := range headers { + buf.WriteString(fmt.Sprintf("%s\r\n", header)) + } + buf.Write(r.UnsafeRawBytes[newLineIndex:]) + r.UnsafeRawBytes = buf.Bytes() + return nil + } + return errors.New("no new line found at the end of host header") + } + + return errors.New("no host header found") +} From 10e1b09bf32bd3e33d9b8c882bc5bea2e7b4adff Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 8 Nov 2021 19:45:01 +0100 Subject: [PATCH 103/146] tidying go modules --- v2/go.mod | 10 +++++++++- v2/go.sum | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/v2/go.mod b/v2/go.mod index 2c003738..39a54ff3 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -3,6 +3,7 @@ module github.com/projectdiscovery/nuclei/v2 go 1.17 require ( + github.com/Ice3man543/nvd v1.0.8 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/akrylysov/pogreb v0.10.1 // indirect github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c @@ -19,6 +20,7 @@ require ( github.com/gosuri/uiprogress v0.0.1 // indirect github.com/itchyny/gojq v0.12.4 github.com/json-iterator/go v1.1.12 + github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.16.1 github.com/logrusorgru/aurora v2.0.3+incompatible @@ -47,10 +49,10 @@ require ( github.com/shirou/gopsutil/v3 v3.21.7 github.com/spaolacci/murmur3 v1.1.0 github.com/spf13/cast v1.4.1 + github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.1 - github.com/weppos/publicsuffix-go v0.15.0 github.com/xanzy/go-gitlab v0.50.3 github.com/ysmood/gson v0.6.4 // indirect github.com/ysmood/leakless v0.7.0 // indirect @@ -66,15 +68,20 @@ require ( moul.io/http2curl v1.0.0 ) +require github.com/weppos/publicsuffix-go v0.15.0 + require ( git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect + github.com/PuerkitoBio/goquery v1.6.0 // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect + github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.1.6 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/eggsampler/acme/v3 v3.2.1 // indirect @@ -99,6 +106,7 @@ require ( github.com/microcosm-cc/bluemonday v1.0.15 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect diff --git a/v2/go.sum b/v2/go.sum index 177689ca..a497178f 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -42,6 +42,7 @@ github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EF github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/Ice3man543/nvd v1.0.8 h1:2CBEgOxyWAkQocnnmEMmRtVPWooPRvcuHFLWj48EM4c= github.com/Ice3man543/nvd v1.0.8/go.mod h1:0DxLJk6revOcJKiZxa2K+rNF/HO1zJO97lqQtXhXfSc= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= @@ -53,6 +54,7 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94= github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -73,6 +75,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= +github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andygrunwald/go-jira v1.14.0 h1:7GT/3qhar2dGJ0kq8w0d63liNyHOnxZsUZ9Pe4+AKBI= github.com/andygrunwald/go-jira v1.14.0/go.mod h1:KMo2f4DgMZA1C9FdImuLc04x4WQhn5derQpnsuBFgqE= @@ -406,6 +409,7 @@ github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0b github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= From 155b8cfed9b66eee291986886ad95fa3c66d9ba4 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 9 Nov 2021 06:00:30 +0530 Subject: [PATCH 104/146] fix #1173: perform matching on all redirect responses instead of final --- v2/pkg/protocols/http/request.go | 121 +++++++++++-------------------- v2/pkg/protocols/http/utils.go | 94 ++++++++++++++---------- 2 files changed, 100 insertions(+), 115 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 3f94c81f..16be32a6 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -400,7 +400,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate return errors.Wrap(err, "could not dump http response") } - var data, redirectedResponse []byte + var dumpedResponse []redirectedResponse // If the status code is HTTP 101, we should not proceed with reading body. if resp.StatusCode != http.StatusSwitchingProtocols { var bodyReader io.Reader @@ -409,7 +409,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } else { bodyReader = resp.Body } - data, err = ioutil.ReadAll(bodyReader) + data, err := ioutil.ReadAll(bodyReader) if err != nil { // Ignore body read due to server misconfiguration errors if stringsutil.ContainsAny(err.Error(), "gzip: invalid header") { @@ -420,96 +420,61 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } resp.Body.Close() - redirectedResponse, err = dumpResponseWithRedirectChain(resp, data) + dumpedResponse, err = dumpResponseWithRedirectChain(resp, data) if err != nil { return errors.Wrap(err, "could not read http response with redirect chain") } } else { - redirectedResponse = dumpedResponseHeaders + dumpedResponse = []redirectedResponse{{fullResponse: dumpedResponseHeaders, headers: dumpedResponseHeaders}} } - // net/http doesn't automatically decompress the response body if an - // encoding has been specified by the user in the request so in case we have to - // manually do it. - dataOrig := data - data, err = handleDecompression(resp, data) - // in case of error use original data - if err != nil { - data = dataOrig - } - - // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) - dumpedResponseBuilder := &bytes.Buffer{} - dumpedResponseBuilder.Write(dumpedResponseHeaders) - dumpedResponseBuilder.Write(data) - dumpedResponse := dumpedResponseBuilder.Bytes() - redirectedResponse = bytes.ReplaceAll(redirectedResponse, dataOrig, data) - - // Decode gbk response content-types - // gb18030 supersedes gb2312 - responseContentType := resp.Header.Get("Content-Type") - if isContentTypeGbk(responseContentType) { - dumpedResponse, err = decodegbk(dumpedResponse) - if err != nil { - return errors.Wrap(err, "could not gbk decode") - } - redirectedResponse, err = decodegbk(redirectedResponse) - if err != nil { - return errors.Wrap(err, "could not gbk decode") + for _, response := range dumpedResponse { + // if nuclei-project is enabled store the response if not previously done + if request.options.ProjectFile != nil && !fromCache { + if err := request.options.ProjectFile.Set(dumpedRequest, resp, response.body); err != nil { + return errors.Wrap(err, "could not store in project file") + } } - // the uncompressed body needs to be decoded to standard utf8 - data, err = decodegbk(data) - if err != nil { - return errors.Wrap(err, "could not gbk decode") + matchedURL := reqURL + if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" { + matchedURL = generatedRequest.rawRequest.FullURL } - } - - // if nuclei-project is enabled store the response if not previously done - if request.options.ProjectFile != nil && !fromCache { - if err := request.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil { - return errors.Wrap(err, "could not store in project file") + if generatedRequest.request != nil { + matchedURL = generatedRequest.request.URL.String() } - } + finalEvent := make(output.InternalEvent) - matchedURL := reqURL - if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" { - matchedURL = generatedRequest.rawRequest.FullURL - } - if generatedRequest.request != nil { - matchedURL = generatedRequest.request.URL.String() - } - finalEvent := make(output.InternalEvent) - - outputEvent := request.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, generatedRequest.meta) - if i := strings.LastIndex(hostname, ":"); i != -1 { - hostname = hostname[:i] - } - outputEvent["curl-command"] = curlCommand - outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) - outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse) - for k, v := range previousEvent { - finalEvent[k] = v - } - for k, v := range outputEvent { - finalEvent[k] = v - } - // Add to history the current request number metadata if asked by the user. - if request.ReqCondition { + outputEvent := request.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) + if i := strings.LastIndex(hostname, ":"); i != -1 { + hostname = hostname[:i] + } + outputEvent["curl-command"] = curlCommand + outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) + for k, v := range previousEvent { + finalEvent[k] = v + } for k, v := range outputEvent { - key := fmt.Sprintf("%s_%d", k, requestCount) - previousEvent[key] = v - finalEvent[key] = v + finalEvent[k] = v } + // Add to history the current request number metadata if asked by the user. + if request.ReqCondition { + for k, v := range outputEvent { + key := fmt.Sprintf("%s_%d", k, requestCount) + previousEvent[key] = v + finalEvent[key] = v + } + } + + event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta + }) + + responseContentType := resp.Header.Get("Content-Type") + dumpResponse(event, request.options, response.fullResponse, formedURL, responseContentType) + + callback(event) } - - event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { - internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta - }) - - dumpResponse(event, request.options, redirectedResponse, formedURL, responseContentType) - - callback(event) return nil } diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go index 89c35a23..1bd25d38 100644 --- a/v2/pkg/protocols/http/utils.go +++ b/v2/pkg/protocols/http/utils.go @@ -10,14 +10,20 @@ import ( "net/http/httputil" "strings" + "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/stringsutil" "golang.org/x/text/encoding/simplifiedchinese" "golang.org/x/text/transform" ) +type redirectedResponse struct { + headers []byte + body []byte + fullResponse []byte +} + // dumpResponseWithRedirectChain dumps a http response with the // complete http redirect chain. // @@ -25,18 +31,22 @@ import ( // and returns the data to the user for matching and viewing in that order. // // Inspired from - https://github.com/ffuf/ffuf/issues/324#issuecomment-719858923 -func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]byte, error) { - redirects := []string{} +func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirectedResponse, error) { + var response []redirectedResponse + respData, err := httputil.DumpResponse(resp, false) if err != nil { return nil, err } - redirectChain := &bytes.Buffer{} - - redirectChain.WriteString(tostring.UnsafeToString(respData)) - redirectChain.Write(body) - redirects = append(redirects, redirectChain.String()) - redirectChain.Reset() + respObj := redirectedResponse{ + headers: respData, + body: body, + fullResponse: bytes.Join([][]byte{respData, body}, []byte{}), + } + if err := normalizeResponseBody(resp, respObj); err != nil { + return nil, err + } + response = append(response, respObj) var redirectResp *http.Response if resp != nil && resp.Request != nil { @@ -52,40 +62,50 @@ func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]byte, er if redirectResp.Body != nil { body, _ = ioutil.ReadAll(redirectResp.Body) } - redirectChain.WriteString(tostring.UnsafeToString(respData)) - if len(body) > 0 { - redirectChain.WriteString(tostring.UnsafeToString(body)) + respObj := redirectedResponse{ + headers: respData, + body: body, + fullResponse: bytes.Join([][]byte{respData, body}, []byte{}), } - redirects = append(redirects, redirectChain.String()) + if err := normalizeResponseBody(redirectResp, respObj); err != nil { + return nil, err + } + response = append(response, respObj) redirectResp = redirectResp.Request.Response - redirectChain.Reset() } - for i := len(redirects) - 1; i >= 0; i-- { - redirectChain.WriteString(redirects[i]) - } - return redirectChain.Bytes(), nil + return response, nil } -// headersToString converts http headers to string -func headersToString(headers http.Header) string { - builder := &strings.Builder{} - - for header, values := range headers { - builder.WriteString(header) - builder.WriteString(": ") - - for i, value := range values { - builder.WriteString(value) - - if i != len(values)-1 { - builder.WriteRune('\n') - builder.WriteString(header) - builder.WriteString(": ") - } - } - builder.WriteRune('\n') +// normalizeResponseBody performs normalization on the http response object. +func normalizeResponseBody(resp *http.Response, response redirectedResponse) error { + var err error + // net/http doesn't automatically decompress the response body if an + // encoding has been specified by the user in the request so in case we have to + // manually do it. + dataOrig := response.body + response.body, err = handleDecompression(resp, response.body) + // in case of error use original data + if err != nil { + response.body = dataOrig } - return builder.String() + response.fullResponse = bytes.ReplaceAll(response.fullResponse, dataOrig, response.body) + + // Decode gbk response content-types + // gb18030 supersedes gb2312 + responseContentType := resp.Header.Get("Content-Type") + if isContentTypeGbk(responseContentType) { + response.fullResponse, err = decodegbk(response.fullResponse) + if err != nil { + return errors.Wrap(err, "could not gbk decode") + } + + // the uncompressed body needs to be decoded to standard utf8 + response.body, err = decodegbk(response.body) + if err != nil { + return errors.Wrap(err, "could not gbk decode") + } + } + return nil } // dump creates a dump of the http request in form of a byte slice From 8396413f9d7b0c92f3df3e43308ec38ba460185e Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 9 Nov 2021 06:10:07 +0530 Subject: [PATCH 105/146] Added integration test for the redirect chain http functionality --- .../http/get-redirects-chain-headers.yaml | 18 ++++++++++++ v2/cmd/integration-test/http.go | 29 +++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 integration_tests/http/get-redirects-chain-headers.yaml diff --git a/integration_tests/http/get-redirects-chain-headers.yaml b/integration_tests/http/get-redirects-chain-headers.yaml new file mode 100644 index 00000000..c4446fec --- /dev/null +++ b/integration_tests/http/get-redirects-chain-headers.yaml @@ -0,0 +1,18 @@ +id: basic-get-redirects-chain-headers + +info: + name: Basic GET Redirects Request With Chain header + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + redirects: true + max-redirects: 3 + matchers: + - type: word + part: header + words: + - "TestRedirectHeaderMatch" \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index a81111c4..f0d335dd 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -35,6 +35,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/self-contained.yaml": &httpRequestSelContained{}, "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{}, "http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{}, + "http/get-redirects-chain-headers.yaml": &httpGetRedirectsChainHeaders{}, } type httpInteractshRequest struct{} @@ -599,3 +600,31 @@ func (h *httpGetCaseInsensitiveCluster) Execute(filesPath string) error { } return nil } + +type httpGetRedirectsChainHeaders struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpGetRedirectsChainHeaders) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + http.Redirect(w, r, "/redirected", http.StatusFound) + }) + router.GET("/redirected", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.Header().Set("Secret", "TestRedirectHeaderMatch") + http.Redirect(w, r, "/final", http.StatusFound) + }) + router.GET("/final", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.Write([]byte("ok")) + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} From 3e2875267c419475fea48fd4c4debf4dba7894e1 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 9 Nov 2021 06:12:36 +0530 Subject: [PATCH 106/146] linter fixes --- v2/cmd/integration-test/http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index f0d335dd..bd8f4ac2 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -614,7 +614,7 @@ func (h *httpGetRedirectsChainHeaders) Execute(filePath string) error { http.Redirect(w, r, "/final", http.StatusFound) }) router.GET("/final", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - w.Write([]byte("ok")) + _, _ = w.Write([]byte("ok")) }) ts := httptest.NewServer(router) defer ts.Close() From a9f586369a263c47628a26591e6ed8e008d55e66 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 9 Nov 2021 06:20:10 +0530 Subject: [PATCH 107/146] Misc fixes to request normalization --- v2/pkg/protocols/http/utils.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go index 1bd25d38..14088eed 100644 --- a/v2/pkg/protocols/http/utils.go +++ b/v2/pkg/protocols/http/utils.go @@ -43,7 +43,7 @@ func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirect body: body, fullResponse: bytes.Join([][]byte{respData, body}, []byte{}), } - if err := normalizeResponseBody(resp, respObj); err != nil { + if err := normalizeResponseBody(resp, &respObj); err != nil { return nil, err } response = append(response, respObj) @@ -67,7 +67,7 @@ func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirect body: body, fullResponse: bytes.Join([][]byte{respData, body}, []byte{}), } - if err := normalizeResponseBody(redirectResp, respObj); err != nil { + if err := normalizeResponseBody(redirectResp, &respObj); err != nil { return nil, err } response = append(response, respObj) @@ -77,7 +77,7 @@ func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirect } // normalizeResponseBody performs normalization on the http response object. -func normalizeResponseBody(resp *http.Response, response redirectedResponse) error { +func normalizeResponseBody(resp *http.Response, response *redirectedResponse) error { var err error // net/http doesn't automatically decompress the response body if an // encoding has been specified by the user in the request so in case we have to From 93060ec0e30eb8ef78acbf5704bf8f62d701f70a Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 9 Nov 2021 08:50:18 +0100 Subject: [PATCH 108/146] adding tests --- v2/pkg/protocols/http/raw/raw_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/v2/pkg/protocols/http/raw/raw_test.go b/v2/pkg/protocols/http/raw/raw_test.go index 0e7cee9b..e661d380 100644 --- a/v2/pkg/protocols/http/raw/raw_test.go +++ b/v2/pkg/protocols/http/raw/raw_test.go @@ -75,3 +75,13 @@ Connection: close`, "https://test.com/test/", true) require.Nil(t, err, "could not parse unsafe request") require.Contains(t, string(request.UnsafeRawBytes), "GET /test/manager/html", "Could not parse unsafe method request path correctly") } + +func TestTryFillCustomHeaders(t *testing.T) { + testValue := "GET /manager/html HTTP/1.1\r\nHost: Test\r\n" + expected := "GET /test/manager/html HTTP/1.1\r\nHost: Test\r\ntest: test\r\n" + request, err := Parse(testValue, "https://test.com/test/", true) + require.Nil(t, err, "could not parse unsafe request") + err = request.TryFillCustomHeaders([]string{"test: test"}) + require.Nil(t, err, "could not add custom headers") + require.Equal(t, expected, string(request.UnsafeRawBytes), "actual value and expected value are different") +} From f2c81ee9a2def6d103161652f2d6181b6a32e33f Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 9 Nov 2021 16:11:11 +0530 Subject: [PATCH 109/146] fixed unsafe test --- integration_tests/http/raw-unsafe-request.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/http/raw-unsafe-request.yaml b/integration_tests/http/raw-unsafe-request.yaml index 0a84b915..e7c45c98 100644 --- a/integration_tests/http/raw-unsafe-request.yaml +++ b/integration_tests/http/raw-unsafe-request.yaml @@ -7,7 +7,7 @@ info: requests: - raw: - - | + - |+ GET / HTTP/1.1 Host: Content-Length: 4 From 9b364080ea506d0cd3d96f14062c2f47ee839746 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 9 Nov 2021 17:55:42 +0530 Subject: [PATCH 110/146] Fixed some edge cases with status codes in redirect chains --- integration_tests/http/get-redirects-chain-headers.yaml | 7 ++++++- v2/cmd/integration-test/integration-test.go | 2 +- v2/pkg/protocols/http/request.go | 2 +- v2/pkg/protocols/http/utils.go | 3 +++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/integration_tests/http/get-redirects-chain-headers.yaml b/integration_tests/http/get-redirects-chain-headers.yaml index c4446fec..51207301 100644 --- a/integration_tests/http/get-redirects-chain-headers.yaml +++ b/integration_tests/http/get-redirects-chain-headers.yaml @@ -11,8 +11,13 @@ requests: - "{{BaseURL}}" redirects: true max-redirects: 3 + matchers-condition: and matchers: - type: word part: header words: - - "TestRedirectHeaderMatch" \ No newline at end of file + - "TestRedirectHeaderMatch" + + - type: status + status: + - 302 \ No newline at end of file diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index 94e6c215..72a81a67 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -51,5 +51,5 @@ func main() { } func errIncorrectResultsCount(results []string) error { - return fmt.Errorf("incorrect number of results %s", strings.Join(results, "\n\t")) + return fmt.Errorf("incorrect number of results \n\t%s", strings.Join(results, "\n\t")) } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 16be32a6..9b5b02a6 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -445,7 +445,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } finalEvent := make(output.InternalEvent) - outputEvent := request.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) + outputEvent := request.responseToDSLMap(response.resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go index 14088eed..5eae57ef 100644 --- a/v2/pkg/protocols/http/utils.go +++ b/v2/pkg/protocols/http/utils.go @@ -22,6 +22,7 @@ type redirectedResponse struct { headers []byte body []byte fullResponse []byte + resp *http.Response } // dumpResponseWithRedirectChain dumps a http response with the @@ -41,6 +42,7 @@ func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirect respObj := redirectedResponse{ headers: respData, body: body, + resp: resp, fullResponse: bytes.Join([][]byte{respData, body}, []byte{}), } if err := normalizeResponseBody(resp, &respObj); err != nil { @@ -65,6 +67,7 @@ func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirect respObj := redirectedResponse{ headers: respData, body: body, + resp: redirectResp, fullResponse: bytes.Join([][]byte{respData, body}, []byte{}), } if err := normalizeResponseBody(redirectResp, &respObj); err != nil { From 4b169fcae1f264a9b895793aaa9cdd02a404877e Mon Sep 17 00:00:00 2001 From: sandeep Date: Wed, 10 Nov 2021 15:09:40 +0530 Subject: [PATCH 111/146] mod update --- v2/go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index e1080d2a..15980fdd 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -64,8 +64,6 @@ require ( moul.io/http2curl v1.0.0 ) -require github.com/weppos/publicsuffix-go v0.15.0 - require ( git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect github.com/PuerkitoBio/goquery v1.6.0 // indirect From 0e46d3e0410f89e4af41099f5cd38fa7f54ae54f Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Wed, 10 Nov 2021 10:00:03 -0600 Subject: [PATCH 112/146] feat: Checking socks5 proxy before launching a scan #1001 (#1225) * Proxy validation and list input support Co-authored-by: Sajad Parra Co-authored-by: sandeep --- README.md | 20 +-- README_CN.md | 3 +- v2/cmd/nuclei/main.go | 6 +- v2/internal/runner/options.go | 25 +--- v2/internal/runner/proxy.go | 123 ++++++++++++++++++ v2/internal/testutils/testutils.go | 3 +- v2/pkg/protocols/headless/engine/engine.go | 4 +- .../protocols/headless/engine/http_client.go | 13 +- .../http/httpclientpool/clientpool.go | 45 +++---- v2/pkg/types/proxy.go | 15 +++ v2/pkg/types/types.go | 6 +- 11 files changed, 184 insertions(+), 79 deletions(-) create mode 100644 v2/internal/runner/proxy.go create mode 100644 v2/pkg/types/proxy.go diff --git a/README.md b/README.md index ca198881..e4626a05 100644 --- a/README.md +++ b/README.md @@ -162,16 +162,16 @@ HEADLESS: -sc, -system-chrome Use local installed chrome browser instead of nuclei installed DEBUG: - -debug show all requests and responses - -debug-req show all sent requests - -debug-resp show all received responses - -proxy, -proxy-url string URL of the HTTP proxy server - -proxy-socks-url string URL of the SOCKS proxy server - -tlog, -trace-log string file to write sent requests trace log - -version show nuclei version - -v, -verbose show verbose output - -vv display extra verbose information - -tv, -templates-version shows the version of the installed nuclei-templates + -debug show all requests and responses + -debug-req show all sent requests + -debug-resp show all received responses + -p, -proxy string[] List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input) + -tlog, -trace-log string file to write sent requests trace log + -elog, -error-log string file to write sent requests error log + -version show nuclei version + -v, -verbose show verbose output + -vv display templates loaded for scan + -tv, -templates-version shows the version of the installed nuclei-templates UPDATE: -update update nuclei engine to the latest released version diff --git a/README_CN.md b/README_CN.md index 04138acd..0c4aa69c 100644 --- a/README_CN.md +++ b/README_CN.md @@ -121,8 +121,7 @@ nuclei -h |templates-version|显示已安装的模板版本|nuclei -templates-version| |v|显示发送请求的详细信息|nuclei -v| |version|显示nuclei的版本号|nuclei -version| -|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080| -|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080| +|proxy|输入代理地址|nuclei -proxy ./proxy.txt| |random-agent|使用随机的UA|nuclei -random-agent| |H|自定义请求头|nuclei -H “x-bug-bounty:hacker”| diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index f4204157..ac176732 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -141,11 +141,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.Debug, "debug", false, "show all requests and responses"), flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"), flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"), - - /* TODO why the separation? http://proxy:port vs socks5://proxy:port etc - TODO should auto-set the HTTP_PROXY variable for the process? */ - flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"), - flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"), + flagSet.NormalizedStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)"), flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), flagSet.StringVarP(&options.ErrorLogFile, "error-log", "elog", "", "file to write sent requests error log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 3866b744..c8376f76 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -3,7 +3,6 @@ package runner import ( "bufio" "errors" - "net/url" "os" "path/filepath" "strings" @@ -24,7 +23,6 @@ func ParseOptions(options *types.Options) { // Read the inputs and configure the logging configureOutput(options) - // Show the user the banner showBanner() @@ -89,15 +87,10 @@ func validateOptions(options *types.Options) error { if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") } - - if err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)"); err != nil { + //loading the proxy server list from file or cli and test the connectivity + if err := loadProxyServers(options); err != nil { return err } - - if err := validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)"); err != nil { - return err - } - if options.Validate { options.Headless = true // required for correct validation of headless templates validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) @@ -114,19 +107,6 @@ func validateOptions(options *types.Options) error { return nil } -func validateProxyURL(proxyURL, message string) error { - if proxyURL != "" && !isValidURL(proxyURL) { - return errors.New(message) - } - - return nil -} - -func isValidURL(urlString string) bool { - _, err := url.Parse(urlString) - return err == nil -} - // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output @@ -172,7 +152,6 @@ func loadResolvers(options *types.Options) { func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) { allGivenTemplatePaths := append(templatePaths, workflowPaths...) - for _, templatePath := range allGivenTemplatePaths { if templatesDirectory != templatePath && filepath.IsAbs(templatePath) { fileInfo, err := os.Stat(templatePath) diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go new file mode 100644 index 00000000..e254472a --- /dev/null +++ b/v2/internal/runner/proxy.go @@ -0,0 +1,123 @@ +package runner + +import ( + "bufio" + "errors" + "fmt" + "net" + "net/url" + "os" + "strings" + "time" + + "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +var proxyURLList []url.URL + +// loadProxyServers load list of proxy servers from file or comma seperated +func loadProxyServers(options *types.Options) error { + if len(options.Proxy) == 0 { + return nil + } + for _, p := range options.Proxy { + if proxyURL, err := validateProxyURL(p); err == nil { + proxyURLList = append(proxyURLList, proxyURL) + } else if fileutil.FileExists(p) { + file, err := os.Open(p) + if err != nil { + return fmt.Errorf("could not open proxy file: %s", err) + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + proxy := scanner.Text() + if strings.TrimSpace(proxy) == "" { + continue + } + if proxyURL, err := validateProxyURL(proxy); err != nil { + return err + } else { + proxyURLList = append(proxyURLList, proxyURL) + } + } + } else { + return fmt.Errorf("invalid proxy file or URL provided for %s", p) + } + } + return processProxyList(options) +} + +func processProxyList(options *types.Options) error { + if len(proxyURLList) == 0 { + return fmt.Errorf("could not find any valid proxy") + } else { + done := make(chan bool) + exitCounter := make(chan bool) + counter := 0 + for _, url := range proxyURLList { + go runProxyConnectivity(url, options, done, exitCounter) + } + for { + select { + case <-done: + { + close(done) + return nil + } + case <-exitCounter: + { + if counter += 1; counter == len(proxyURLList) { + return errors.New("no reachable proxy found") + } + } + } + } + } +} + +func runProxyConnectivity(proxyURL url.URL, options *types.Options, done chan bool, exitCounter chan bool) { + if err := testProxyConnection(proxyURL, options.Timeout); err == nil { + if types.ProxyURL == "" && types.ProxySocksURL == "" { + assignProxyURL(proxyURL, options) + done <- true + } + } + exitCounter <- true +} + +func testProxyConnection(proxyURL url.URL, timeoutDelay int) error { + timeout := time.Duration(timeoutDelay) * time.Second + _, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), timeout) + if err != nil { + return err + } + return nil +} + +func assignProxyURL(proxyURL url.URL, options *types.Options) { + os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String()) + if proxyURL.Scheme == types.HTTP || proxyURL.Scheme == types.HTTPS { + types.ProxyURL = proxyURL.String() + types.ProxySocksURL = "" + gologger.Verbose().Msgf("Using %s as proxy server", proxyURL.String()) + } else if proxyURL.Scheme == types.SOCKS5 { + types.ProxyURL = "" + types.ProxySocksURL = proxyURL.String() + gologger.Verbose().Msgf("Using %s as socket proxy server", proxyURL.String()) + } +} + +func validateProxyURL(proxy string) (url.URL, error) { + if url, err := url.Parse(proxy); err == nil && isSupportedProtocol(url.Scheme) { + return *url, nil + } + return url.URL{}, errors.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)") +} + +//isSupportedProtocol checks given protocols are supported +func isSupportedProtocol(value string) bool { + return value == types.HTTP || value == types.HTTPS || value == types.SOCKS5 +} diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index 79e42642..a26a1a1c 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -51,8 +51,7 @@ var DefaultOptions = &types.Options{ Targets: []string{}, TargetsFilePath: "", Output: "", - ProxyURL: "", - ProxySocksURL: "", + Proxy: []string{}, TemplatesDirectory: "", TraceLogFile: "", Templates: []string{}, diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index a506cbda..a4259580 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -63,8 +63,8 @@ func New(options *types.Options) (*Browser, error) { } else { chromeLauncher = chromeLauncher.Headless(true) } - if options.ProxyURL != "" { - chromeLauncher = chromeLauncher.Proxy(options.ProxyURL) + if types.ProxyURL != "" { + chromeLauncher = chromeLauncher.Proxy(types.ProxyURL) } launcherURL, err := chromeLauncher.Launch() if err != nil { diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index 4ae14cf1..20b7840a 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -4,13 +4,14 @@ import ( "context" "crypto/tls" "fmt" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" "net" "net/http" "net/http/cookiejar" "net/url" "time" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/types" "golang.org/x/net/proxy" @@ -36,15 +37,13 @@ func newhttpClient(options *types.Options) *http.Client { MaxConnsPerHost: 500, TLSClientConfig: tlsConfig, } - - if options.ProxyURL != "" { - if proxyURL, err := url.Parse(options.ProxyURL); err == nil { + if types.ProxyURL != "" { + if proxyURL, err := url.Parse(types.ProxyURL); err == nil { transport.Proxy = http.ProxyURL(proxyURL) } - } else if options.ProxySocksURL != "" { + } else if types.ProxySocksURL != "" { var proxyAuth *proxy.Auth - - socksURL, proxyErr := url.Parse(options.ProxySocksURL) + socksURL, proxyErr := url.Parse(types.ProxySocksURL) if proxyErr == nil { proxyAuth = &proxy.Auth{} proxyAuth.User = socksURL.User.Username() diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 2b7dc024..2817b459 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -4,7 +4,6 @@ import ( "context" "crypto/tls" "fmt" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" "net" "net/http" "net/http/cookiejar" @@ -14,6 +13,8 @@ import ( "sync" "time" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" + "github.com/pkg/errors" "golang.org/x/net/proxy" "golang.org/x/net/publicsuffix" @@ -129,9 +130,8 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl return client, nil } poolMutex.RUnlock() - - if options.ProxyURL != "" { - proxyURL, err = url.Parse(options.ProxyURL) + if types.ProxyURL != "" { + proxyURL, err = url.Parse(types.ProxyURL) } if err != nil { return nil, err @@ -179,27 +179,24 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl TLSClientConfig: tlsConfig, DisableKeepAlives: disableKeepAlives, } - - // Attempts to overwrite the dial function with the socks proxied version - if options.ProxySocksURL != "" { - var proxyAuth *proxy.Auth - - socksURL, proxyErr := url.Parse(options.ProxySocksURL) - if proxyErr == nil { - proxyAuth = &proxy.Auth{} - proxyAuth.User = socksURL.User.Username() - proxyAuth.Password, _ = socksURL.User.Password() - } - dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) - dc := dialer.(interface { - DialContext(ctx context.Context, network, addr string) (net.Conn, error) - }) - if proxyErr == nil { - transport.DialContext = dc.DialContext - } - } if proxyURL != nil { - transport.Proxy = http.ProxyURL(proxyURL) + // Attempts to overwrite the dial function with the socks proxied version + if proxyURL.Scheme == types.SOCKS5 { + var proxyAuth *proxy.Auth = &proxy.Auth{} + proxyAuth.User = proxyURL.User.Username() + proxyAuth.Password, _ = proxyURL.User.Password() + + dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), proxyAuth, proxy.Direct) + + dc := dialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + if proxyErr == nil { + transport.DialContext = dc.DialContext + } + } else { + transport.Proxy = http.ProxyURL(proxyURL) + } } var jar *cookiejar.Jar diff --git a/v2/pkg/types/proxy.go b/v2/pkg/types/proxy.go new file mode 100644 index 00000000..a45b4eca --- /dev/null +++ b/v2/pkg/types/proxy.go @@ -0,0 +1,15 @@ +package types + +const ( + HTTP_PROXY_ENV = "HTTP_PROXY" + SOCKS5 = "socks5" + HTTP = "http" + HTTPS = "https" +) + +var ( + // ProxyURL is the URL for the proxy server + ProxyURL string + // ProxySocksURL is the URL for the proxy socks server + ProxySocksURL string +) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index a40e460a..bb8e659f 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -58,10 +58,8 @@ type Options struct { TargetsFilePath string // Output is the file to write found results to. Output string - // ProxyURL is the URL for the proxy server - ProxyURL string - // ProxySocksURL is the URL for the proxy socks server - ProxySocksURL string + // List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input) + Proxy goflags.NormalizedStringSlice // TemplatesDirectory is the directory to use for storing templates TemplatesDirectory string // TraceLogFile specifies a file to write with the trace of all requests From 56b7d78fedb2eb946e3e7a8c095388f34ef41efe Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 10 Nov 2021 18:04:53 +0100 Subject: [PATCH 113/146] Fixing http test using local http mock server --- .../offlinehttp/read_response_test.go | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/offlinehttp/read_response_test.go b/v2/pkg/protocols/offlinehttp/read_response_test.go index 4a75d726..f47b345d 100644 --- a/v2/pkg/protocols/offlinehttp/read_response_test.go +++ b/v2/pkg/protocols/offlinehttp/read_response_test.go @@ -1,12 +1,15 @@ package offlinehttp import ( + "fmt" "io/ioutil" "net/http" + "net/http/httptest" "net/http/httputil" "testing" "time" + "github.com/julienschmidt/httprouter" "github.com/stretchr/testify/require" ) @@ -154,11 +157,30 @@ Server: Google Frontend } t.Run("test-live-response-with-content-length", func(t *testing.T) { + var ts *httptest.Server + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + w.Header().Add("Server", "Google Frontend") + fmt.Fprintf(w, "%s", ` + + + Firing Range + + +

Version 0.48

+

What is the Firing Range?

+

+ + `) + })) + ts = httptest.NewServer(router) + defer ts.Close() + client := &http.Client{ Timeout: 3 * time.Second, } - data, err := client.Get("https://golang.org/doc/install") + data, err := client.Get(ts.URL) require.Nil(t, err, "could not dial url") defer data.Body.Close() From 09b5fb11e8799e465aaa2b00397ddcb789f49e1f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 10 Nov 2021 18:11:42 +0100 Subject: [PATCH 114/146] Better error handling for return value of helper http library (#1239) --- v2/pkg/protocols/http/request.go | 38 +++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 9b5b02a6..3958d6c7 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -99,16 +99,16 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output. mutex := &sync.Mutex{} for { generatedHttpRequest, err := generator.Make(reqURL, dynamicValues) - if err == io.EOF { - break + if err != nil { + if err == io.EOF { + break + } + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + return err } if reqURL == "" { reqURL = generatedHttpRequest.URL() } - if err != nil { - request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) - return err - } swg.Add() go func(httpRequest *generatedRequest) { defer swg.Done() @@ -162,18 +162,17 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous mutex := &sync.Mutex{} for { generatedHttpRequest, err := generator.Make(reqURL, dynamicValues) - if err == io.EOF { - break + if err != nil { + if err == io.EOF { + break + } + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + return err } if reqURL == "" { reqURL = generatedHttpRequest.URL() } - if err != nil { - request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) - return err - } generatedHttpRequest.pipelinedClient = pipeClient - swg.Add() go func(httpRequest *generatedRequest) { defer swg.Done() @@ -216,17 +215,16 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators) generatedHttpRequest, err := generator.Make(reqURL, dynamicValues) - if err == io.EOF { - break + if err != nil { + if err == io.EOF { + break + } + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + return err } if reqURL == "" { reqURL = generatedHttpRequest.URL() } - if err != nil { - request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) - return err - } - request.dynamicValues = generatedHttpRequest.dynamicValues // Check if hosts just keep erroring if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) { From ac75d9aa9bfe6f89576482dceb2febb0e5f69f5f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 10 Nov 2021 18:12:49 +0100 Subject: [PATCH 115/146] Improving error handling in client certificate library (#1237) --- v2/pkg/protocols/headless/engine/engine.go | 7 ++++++- v2/pkg/protocols/headless/engine/http_client.go | 10 +++++++--- v2/pkg/protocols/http/httpclientpool/clientpool.go | 5 ++++- v2/pkg/protocols/utils/utils.go | 12 ++++++------ 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index a4259580..c435bef4 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -88,7 +88,12 @@ func New(options *types.Options) (*Browser, error) { if customAgent == "" { customAgent = uarand.GetRandom() } - httpclient := newhttpClient(options) + + httpclient, err := newhttpClient(options) + if err != nil { + return nil, err + } + engine := &Browser{ tempDir: dataStore, customAgent: customAgent, diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index 20b7840a..1f40f98f 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -18,7 +18,7 @@ import ( ) // newhttpClient creates a new http client for headless communication with a timeout -func newhttpClient(options *types.Options) *http.Client { +func newhttpClient(options *types.Options) (*http.Client, error) { dialer := protocolstate.Dialer // Set the base TLS configuration definition @@ -28,7 +28,11 @@ func newhttpClient(options *types.Options) *http.Client { } // Add the client certificate authentication to the request if it's configured - tlsConfig = utils.AddConfiguredClientCertToRequest(tlsConfig, options) + var err error + tlsConfig, err = utils.AddConfiguredClientCertToRequest(tlsConfig, options) + if err != nil { + return nil, err + } transport := &http.Transport{ DialContext: dialer.Dial, @@ -70,5 +74,5 @@ func newhttpClient(options *types.Options) *http.Client { }, } - return httpclient + return httpclient, nil } diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 2817b459..191ca3cf 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -169,7 +169,10 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl } // Add the client certificate authentication to the request if it's configured - tlsConfig = utils.AddConfiguredClientCertToRequest(tlsConfig, options) + tlsConfig, err = utils.AddConfiguredClientCertToRequest(tlsConfig, options) + if err != nil { + return nil, errors.Wrap(err, "could not create client certificate") + } transport := &http.Transport{ DialContext: Dialer.Dial, diff --git a/v2/pkg/protocols/utils/utils.go b/v2/pkg/protocols/utils/utils.go index 8d43fd47..d139f128 100644 --- a/v2/pkg/protocols/utils/utils.go +++ b/v2/pkg/protocols/utils/utils.go @@ -3,13 +3,13 @@ package utils import ( "crypto/tls" "crypto/x509" - "github.com/projectdiscovery/nuclei/v2/pkg/types" "io/ioutil" - "log" + + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // AddConfiguredClientCertToRequest adds the client certificate authentication to the tls.Config object and returns it -func AddConfiguredClientCertToRequest(tlsConfig *tls.Config, options *types.Options) *tls.Config { +func AddConfiguredClientCertToRequest(tlsConfig *tls.Config, options *types.Options) (*tls.Config, error) { // Build the TLS config with the client certificate if it has been configured with the appropriate options. // Only one of the options needs to be checked since the validation checks in main.go ensure that all three // files are set if any of the client certification configuration options are. @@ -17,18 +17,18 @@ func AddConfiguredClientCertToRequest(tlsConfig *tls.Config, options *types.Opti // Load the client certificate using the PEM encoded client certificate and the private key file cert, err := tls.LoadX509KeyPair(options.ClientCertFile, options.ClientKeyFile) if err != nil { - log.Fatal(err) + return nil, err } tlsConfig.Certificates = []tls.Certificate{cert} // Load the certificate authority PEM certificate into the TLS configuration caCert, err := ioutil.ReadFile(options.ClientCAFile) if err != nil { - log.Fatal(err) + return nil, err } caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) tlsConfig.RootCAs = caCertPool } - return tlsConfig + return tlsConfig, nil } From 1d74f6ada2cdf035789e41c416f1214786241c25 Mon Sep 17 00:00:00 2001 From: sullo Date: Wed, 10 Nov 2021 12:55:22 -0500 Subject: [PATCH 116/146] Typo (#1242) --- v2/pkg/protocols/dns/operators.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 0cf54988..d6be2f7d 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -14,7 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" ) -// Match matches a generic data response again a given matcher +// Match matches a generic data response against a given matcher func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { item, ok := request.getMatchPart(matcher.Part, data) if !ok { From 90a0502b60d52e15faf8971975824964b5784709 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 11 Nov 2021 17:30:25 +0530 Subject: [PATCH 117/146] Misc fixes to default part --- v2/pkg/operators/extractors/compile.go | 5 ----- v2/pkg/operators/matchers/compile.go | 2 +- v2/pkg/protocols/http/operators.go | 3 +++ 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/v2/pkg/operators/extractors/compile.go b/v2/pkg/operators/extractors/compile.go index 07306e20..56771667 100644 --- a/v2/pkg/operators/extractors/compile.go +++ b/v2/pkg/operators/extractors/compile.go @@ -42,11 +42,6 @@ func (e *Extractor) CompileExtractors() error { e.jsonCompiled = append(e.jsonCompiled, compiled) } - // Set up the part of the request to match, if any. - if e.Part == "" { - e.Part = "body" - } - if e.CaseInsensitive { if e.Type != "kval" { return fmt.Errorf("case-insensitive flag is supported only for 'kval' extractors (not '%s')", e.Type) diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index 63b3edf5..7c7c5555 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -31,7 +31,7 @@ func (m *Matcher) CompileMatchers() error { } // By default, match on body if user hasn't provided any specific items if m.Part == "" { - m.Part = "body" + m.Part = "response" } // Compile the regexes diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 8d7c4b72..868fd011 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -76,6 +76,9 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto // getMatchPart returns the match part honoring "all" matchers + others. func (request *Request) getMatchPart(part string, data output.InternalEvent) (string, bool) { + if part == "" { + part = "body" + } if part == "header" { part = "all_headers" } From cdb08e08793f6f5d41de30ebf0f18b072e79ffbc Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 12 Nov 2021 04:44:02 +0530 Subject: [PATCH 118/146] misc fixes to websocket --- v2/pkg/protocols/websocket/websocket.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index cdc40cc0..47a87a78 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -172,7 +172,11 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam payloadValues["Hostname"] = parsed.Host payloadValues["Host"] = parsed.Hostname() payloadValues["Scheme"] = parsed.Scheme - payloadValues["Path"] = parsed.Path + requestPath := parsed.Path + if values := parsed.Query(); len(values) > 0 { + requestPath = requestPath + "?" + values.Encode() + } + payloadValues["Path"] = requestPath requestOptions := request.options for key, value := range request.Headers { From dc0c568cb8a8d8fd58539f437d3d660e331812b8 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 11 Nov 2021 23:16:47 +0000 Subject: [PATCH 119/146] Auto Generate Syntax Docs + JSONSchema [Thu Nov 11 23:16:47 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 353 +++++++++++++++++++++++++++++- nuclei-jsonschema.json | 165 +++++++++++++- v2/pkg/templates/templates_doc.go | 233 +++++++++++++++++--- 3 files changed, 707 insertions(+), 44 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 3d16bda7..d87ac890 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -219,6 +219,32 @@ Headless contains the headless request to make in the template.

+ssl []ssl.Request + +
+
+ +SSL contains the SSL request to make in the template. + +
+ +
+ +
+ +websocket []websocket.Request + +
+
+ +Websocket contains the Websocket request to make in the template. + +
+ +
+ +
+ workflows []workflows.WorkflowTemplate
@@ -829,7 +855,7 @@ in a combined manner allowing multirequest based matchers.
-attack string +attack generators.AttackTypeHolder
@@ -1238,6 +1264,10 @@ Appears in: - headless.Request.matchers +- ssl.Request.matchers + +- websocket.Request.matchers +
@@ -1594,6 +1624,10 @@ Appears in: - headless.Request.extractors +- ssl.Request.extractors + +- websocket.Request.extractors +
@@ -1877,6 +1911,22 @@ Valid values: +## generators.AttackTypeHolder +AttackTypeHolder is used to hold internal type of the protocol + +Appears in: + + +- http.Request.attack + +- network.Request.attack + +- websocket.Request.attack + + + + + ## dns.Request Request contains a DNS protocol request to be made from a template @@ -1998,7 +2048,7 @@ name: '{{FQDN}}'
-Type is the type of DNS request to make. +RequestType is the type of DNS request to make. Valid values: @@ -2358,7 +2408,7 @@ host:
-attack string +attack generators.AttackTypeHolder
@@ -2852,6 +2902,303 @@ Valid values: +## ssl.Request +Request is a request for the SSL protocol + +Appears in: + + +- Template.ssl + + + +
+ +
+ +matchers []matchers.Matcher + +
+
+ +Matchers contains the detection mechanism for the request to identify +whether the request was successful by doing pattern matching +on request/responses. + +Multiple matchers can be combined with `matcher-condition` flag +which accepts either `and` or `or` as argument. + +
+ +
+ +
+ +extractors []extractors.Extractor + +
+
+ +Extractors contains the extraction mechanism for the request to identify +and extract parts of the response. + +
+ +
+ +
+ +matchers-condition string + +
+
+ +MatchersCondition is the condition between the matchers. Default is OR. + + +Valid values: + + + - and + + - or +
+ +
+ +
+ +address string + +
+
+ +Address contains address for the request + +
+ +
+ + + + + +## websocket.Request +Request is a request for the Websocket protocol + +Appears in: + + +- Template.websocket + + + +
+ +
+ +matchers []matchers.Matcher + +
+
+ +Matchers contains the detection mechanism for the request to identify +whether the request was successful by doing pattern matching +on request/responses. + +Multiple matchers can be combined with `matcher-condition` flag +which accepts either `and` or `or` as argument. + +
+ +
+ +
+ +extractors []extractors.Extractor + +
+
+ +Extractors contains the extraction mechanism for the request to identify +and extract parts of the response. + +
+ +
+ +
+ +matchers-condition string + +
+
+ +MatchersCondition is the condition between the matchers. Default is OR. + + +Valid values: + + + - and + + - or +
+ +
+ +
+ +address string + +
+
+ +Address contains address for the request + +
+ +
+ +
+ +inputs []websocket.Input + +
+
+ +Inputs contains inputs for the websocket protocol + +
+ +
+ +
+ +headers map[string]string + +
+
+ +Headers contains headers for the request. + +
+ +
+ +
+ +attack generators.AttackTypeHolder + +
+
+ +Attack is the type of payload combinations to perform. + +Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates +permutations and combinations for all payloads. + + +Valid values: + + + - sniper + + - pitchfork + + - clusterbomb +
+ +
+ +
+ +payloads map[string]interface{} + +
+
+ +Payloads contains any payloads for the current request. + +Payloads support both key-values combinations where a list +of payloads is provided, or optionally a single file can also +be provided as payload which will be read on run-time. + +
+ +
+ + + + + +## websocket.Input + +Appears in: + + +- websocket.Request.inputs + + + +
+ +
+ +data string + +
+
+ +Data is the data to send as the input. + +It supports DSL Helper Functions as well as normal expressions. + + + +Examples: + + +```yaml +data: TEST +``` + +```yaml +data: hex_decode('50494e47') +``` + + +
+ +
+ +
+ +name string + +
+
+ +Name is the optional name of the data read to provide matching on. + + + +Examples: + + +```yaml +name: prefix +``` + + +
+ +
+ + + + + ## workflows.WorkflowTemplate Appears in: diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 4f22c386..517876ea 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -308,6 +308,16 @@ "additionalProperties": false, "type": "object" }, + "generators.AttackTypeHolder": { + "enum": [ + "batteringram", + "pitchfork", + "clusterbomb" + ], + "type": "string", + "title": "type of the attack", + "description": "Type of the attack" + }, "dns.Request": { "properties": { "matchers": { @@ -615,12 +625,8 @@ "description": "Optional name for the HTTP Request" }, "attack": { - "enum": [ - "batteringram", - "pitchfork", - "clusterbomb" - ], - "type": "string", + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/generators.AttackTypeHolder", "title": "attack is the payload combination", "description": "Attack is the type of payload combinations to perform" }, @@ -786,12 +792,7 @@ "description": "Host to send network requests to" }, "attack": { - "enum": [ - "batteringram", - "pitchfork", - "clusterbomb" - ], - "type": "string", + "$ref": "#/definitions/generators.AttackTypeHolder", "title": "attack is the payload combination", "description": "Attack is the type of payload combinations to perform" }, @@ -853,6 +854,128 @@ "additionalProperties": false, "type": "object" }, + "ssl.Request": { + "properties": { + "matchers": { + "items": { + "$ref": "#/definitions/matchers.Matcher" + }, + "type": "array", + "title": "matchers to run on response", + "description": "Detection mechanism to identify whether the request was successful by doing pattern matching" + }, + "extractors": { + "items": { + "$ref": "#/definitions/extractors.Extractor" + }, + "type": "array", + "title": "extractors to run on response", + "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response" + }, + "matchers-condition": { + "enum": [ + "and", + "or" + ], + "type": "string", + "title": "condition between the matchers", + "description": "Conditions between the matchers" + }, + "address": { + "type": "string", + "title": "address for the ssl request", + "description": "Address contains address for the request" + } + }, + "additionalProperties": false, + "type": "object" + }, + "websocket.Input": { + "properties": { + "data": { + "type": "string", + "title": "data to send as input", + "description": "Data is the data to send as the input" + }, + "name": { + "type": "string", + "title": "optional name for data read", + "description": "Optional name of the data read to provide matching on" + } + }, + "additionalProperties": false, + "type": "object" + }, + "websocket.Request": { + "properties": { + "matchers": { + "items": { + "$ref": "#/definitions/matchers.Matcher" + }, + "type": "array", + "title": "matchers to run on response", + "description": "Detection mechanism to identify whether the request was successful by doing pattern matching" + }, + "extractors": { + "items": { + "$ref": "#/definitions/extractors.Extractor" + }, + "type": "array", + "title": "extractors to run on response", + "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response" + }, + "matchers-condition": { + "enum": [ + "and", + "or" + ], + "type": "string", + "title": "condition between the matchers", + "description": "Conditions between the matchers" + }, + "address": { + "type": "string", + "title": "address for the websocket request", + "description": "Address contains address for the request" + }, + "inputs": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/websocket.Input" + }, + "type": "array", + "title": "inputs for the websocket request", + "description": "Inputs contains any input/output for the current request" + }, + "headers": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object", + "title": "headers contains the request headers", + "description": "Headers contains headers for the request" + }, + "attack": { + "$ref": "#/definitions/generators.AttackTypeHolder", + "title": "attack is the payload combination", + "description": "Attack is the type of payload combinations to perform" + }, + "payloads": { + "patternProperties": { + ".*": { + "additionalProperties": true + } + }, + "type": "object", + "title": "payloads for the webosocket request", + "description": "Payloads contains any payloads for the current request" + } + }, + "additionalProperties": false, + "type": "object" + }, "templates.Template": { "required": [ "id", @@ -919,6 +1042,24 @@ "title": "headless requests to make", "description": "Headless requests to make for the template" }, + "ssl": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/ssl.Request" + }, + "type": "array", + "title": "ssl requests to make", + "description": "SSL requests to make for the template" + }, + "websocket": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/websocket.Request" + }, + "type": "array", + "title": "websocket requests to make", + "description": "Websocket requests to make for the template" + }, "workflows": { "items": { "$schema": "http://json-schema.org/draft-04/schema#", diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index ca416b3b..b24ccdaa 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -9,29 +9,33 @@ import ( ) var ( - TemplateDoc encoder.Doc - MODELInfoDoc encoder.Doc - STRINGSLICEStringSliceDoc encoder.Doc - SEVERITYHolderDoc encoder.Doc - MODELClassificationDoc encoder.Doc - HTTPRequestDoc encoder.Doc - MATCHERSMatcherDoc encoder.Doc - EXTRACTORSExtractorDoc encoder.Doc - DNSRequestDoc encoder.Doc - FILERequestDoc encoder.Doc - NETWORKRequestDoc encoder.Doc - NETWORKInputDoc encoder.Doc - HEADLESSRequestDoc encoder.Doc - ENGINEActionDoc encoder.Doc - WORKFLOWSWorkflowTemplateDoc encoder.Doc - WORKFLOWSMatcherDoc encoder.Doc + TemplateDoc encoder.Doc + MODELInfoDoc encoder.Doc + STRINGSLICEStringSliceDoc encoder.Doc + SEVERITYHolderDoc encoder.Doc + MODELClassificationDoc encoder.Doc + HTTPRequestDoc encoder.Doc + MATCHERSMatcherDoc encoder.Doc + EXTRACTORSExtractorDoc encoder.Doc + GENERATORSAttackTypeHolderDoc encoder.Doc + DNSRequestDoc encoder.Doc + FILERequestDoc encoder.Doc + NETWORKRequestDoc encoder.Doc + NETWORKInputDoc encoder.Doc + HEADLESSRequestDoc encoder.Doc + ENGINEActionDoc encoder.Doc + SSLRequestDoc encoder.Doc + WEBSOCKETRequestDoc encoder.Doc + WEBSOCKETInputDoc encoder.Doc + WORKFLOWSWorkflowTemplateDoc encoder.Doc + WORKFLOWSMatcherDoc encoder.Doc ) func init() { TemplateDoc.Type = "Template" TemplateDoc.Comments[encoder.LineComment] = " Template is a YAML input file which defines all the requests and" TemplateDoc.Description = "Template is a YAML input file which defines all the requests and\n other metadata for a template." - TemplateDoc.Fields = make([]encoder.Doc, 9) + TemplateDoc.Fields = make([]encoder.Doc, 11) TemplateDoc.Fields[0].Name = "id" TemplateDoc.Fields[0].Type = "string" TemplateDoc.Fields[0].Note = "" @@ -79,16 +83,26 @@ func init() { TemplateDoc.Fields[6].Note = "" TemplateDoc.Fields[6].Description = "Headless contains the headless request to make in the template." TemplateDoc.Fields[6].Comments[encoder.LineComment] = "Headless contains the headless request to make in the template." - TemplateDoc.Fields[7].Name = "workflows" - TemplateDoc.Fields[7].Type = "[]workflows.WorkflowTemplate" + TemplateDoc.Fields[7].Name = "ssl" + TemplateDoc.Fields[7].Type = "[]ssl.Request" TemplateDoc.Fields[7].Note = "" - TemplateDoc.Fields[7].Description = "Workflows is a list of workflows to execute for a template." - TemplateDoc.Fields[7].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template." - TemplateDoc.Fields[8].Name = "self-contained" - TemplateDoc.Fields[8].Type = "bool" + TemplateDoc.Fields[7].Description = "SSL contains the SSL request to make in the template." + TemplateDoc.Fields[7].Comments[encoder.LineComment] = "SSL contains the SSL request to make in the template." + TemplateDoc.Fields[8].Name = "websocket" + TemplateDoc.Fields[8].Type = "[]websocket.Request" TemplateDoc.Fields[8].Note = "" - TemplateDoc.Fields[8].Description = "Self Contained marks Requests for the template as self-contained" - TemplateDoc.Fields[8].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained" + TemplateDoc.Fields[8].Description = "Websocket contains the Websocket request to make in the template." + TemplateDoc.Fields[8].Comments[encoder.LineComment] = "Websocket contains the Websocket request to make in the template." + TemplateDoc.Fields[9].Name = "workflows" + TemplateDoc.Fields[9].Type = "[]workflows.WorkflowTemplate" + TemplateDoc.Fields[9].Note = "" + TemplateDoc.Fields[9].Description = "Workflows is a list of workflows to execute for a template." + TemplateDoc.Fields[9].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template." + TemplateDoc.Fields[10].Name = "self-contained" + TemplateDoc.Fields[10].Type = "bool" + TemplateDoc.Fields[10].Note = "" + TemplateDoc.Fields[10].Description = "Self Contained marks Requests for the template as self-contained" + TemplateDoc.Fields[10].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained" MODELInfoDoc.Type = "model.Info" MODELInfoDoc.Comments[encoder.LineComment] = " Info contains metadata information about a template" @@ -320,7 +334,7 @@ func init() { HTTPRequestDoc.Fields[6].Description = "Name is the optional name of the request.\n\nIf a name is specified, all the named request in a template can be matched upon\nin a combined manner allowing multirequest based matchers." HTTPRequestDoc.Fields[6].Comments[encoder.LineComment] = "Name is the optional name of the request." HTTPRequestDoc.Fields[7].Name = "attack" - HTTPRequestDoc.Fields[7].Type = "string" + HTTPRequestDoc.Fields[7].Type = "generators.AttackTypeHolder" HTTPRequestDoc.Fields[7].Note = "" HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nbatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." HTTPRequestDoc.Fields[7].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." @@ -472,6 +486,14 @@ func init() { TypeName: "headless.Request", FieldName: "matchers", }, + { + TypeName: "ssl.Request", + FieldName: "matchers", + }, + { + TypeName: "websocket.Request", + FieldName: "matchers", + }, } MATCHERSMatcherDoc.Fields = make([]encoder.Doc, 13) MATCHERSMatcherDoc.Fields[0].Name = "type" @@ -609,6 +631,14 @@ func init() { TypeName: "headless.Request", FieldName: "extractors", }, + { + TypeName: "ssl.Request", + FieldName: "extractors", + }, + { + TypeName: "websocket.Request", + FieldName: "extractors", + }, } EXTRACTORSExtractorDoc.Fields = make([]encoder.Doc, 11) EXTRACTORSExtractorDoc.Fields[0].Name = "name" @@ -697,6 +727,25 @@ func init() { "true", } + GENERATORSAttackTypeHolderDoc.Type = "generators.AttackTypeHolder" + GENERATORSAttackTypeHolderDoc.Comments[encoder.LineComment] = " AttackTypeHolder is used to hold internal type of the protocol" + GENERATORSAttackTypeHolderDoc.Description = "AttackTypeHolder is used to hold internal type of the protocol" + GENERATORSAttackTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "http.Request", + FieldName: "attack", + }, + { + TypeName: "network.Request", + FieldName: "attack", + }, + { + TypeName: "websocket.Request", + FieldName: "attack", + }, + } + GENERATORSAttackTypeHolderDoc.Fields = make([]encoder.Doc, 0) + DNSRequestDoc.Type = "dns.Request" DNSRequestDoc.Comments[encoder.LineComment] = " Request contains a DNS protocol request to be made from a template" DNSRequestDoc.Description = "Request contains a DNS protocol request to be made from a template" @@ -743,8 +792,8 @@ func init() { DNSRequestDoc.Fields[5].Name = "type" DNSRequestDoc.Fields[5].Type = "string" DNSRequestDoc.Fields[5].Note = "" - DNSRequestDoc.Fields[5].Description = "Type is the type of DNS request to make." - DNSRequestDoc.Fields[5].Comments[encoder.LineComment] = "Type is the type of DNS request to make." + DNSRequestDoc.Fields[5].Description = "RequestType is the type of DNS request to make." + DNSRequestDoc.Fields[5].Comments[encoder.LineComment] = "RequestType is the type of DNS request to make." DNSRequestDoc.Fields[5].Values = []string{ "A", "NS", @@ -875,7 +924,7 @@ func init() { NETWORKRequestDoc.Fields[1].AddExample("", []string{"{{Hostname}}"}) NETWORKRequestDoc.Fields[2].Name = "attack" - NETWORKRequestDoc.Fields[2].Type = "string" + NETWORKRequestDoc.Fields[2].Type = "generators.AttackTypeHolder" NETWORKRequestDoc.Fields[2].Note = "" NETWORKRequestDoc.Fields[2].Description = "Attack is the type of payload combinations to perform.\n\nBatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." NETWORKRequestDoc.Fields[2].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." @@ -1065,6 +1114,128 @@ func init() { "sleep", } + SSLRequestDoc.Type = "ssl.Request" + SSLRequestDoc.Comments[encoder.LineComment] = " Request is a request for the SSL protocol" + SSLRequestDoc.Description = "Request is a request for the SSL protocol" + SSLRequestDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "Template", + FieldName: "ssl", + }, + } + SSLRequestDoc.Fields = make([]encoder.Doc, 4) + SSLRequestDoc.Fields[0].Name = "matchers" + SSLRequestDoc.Fields[0].Type = "[]matchers.Matcher" + SSLRequestDoc.Fields[0].Note = "" + SSLRequestDoc.Fields[0].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument." + SSLRequestDoc.Fields[0].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify" + SSLRequestDoc.Fields[1].Name = "extractors" + SSLRequestDoc.Fields[1].Type = "[]extractors.Extractor" + SSLRequestDoc.Fields[1].Note = "" + SSLRequestDoc.Fields[1].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response." + SSLRequestDoc.Fields[1].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify" + SSLRequestDoc.Fields[2].Name = "matchers-condition" + SSLRequestDoc.Fields[2].Type = "string" + SSLRequestDoc.Fields[2].Note = "" + SSLRequestDoc.Fields[2].Description = "MatchersCondition is the condition between the matchers. Default is OR." + SSLRequestDoc.Fields[2].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR." + SSLRequestDoc.Fields[2].Values = []string{ + "and", + "or", + } + SSLRequestDoc.Fields[3].Name = "address" + SSLRequestDoc.Fields[3].Type = "string" + SSLRequestDoc.Fields[3].Note = "" + SSLRequestDoc.Fields[3].Description = "Address contains address for the request" + SSLRequestDoc.Fields[3].Comments[encoder.LineComment] = "Address contains address for the request" + + WEBSOCKETRequestDoc.Type = "websocket.Request" + WEBSOCKETRequestDoc.Comments[encoder.LineComment] = " Request is a request for the Websocket protocol" + WEBSOCKETRequestDoc.Description = "Request is a request for the Websocket protocol" + WEBSOCKETRequestDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "Template", + FieldName: "websocket", + }, + } + WEBSOCKETRequestDoc.Fields = make([]encoder.Doc, 8) + WEBSOCKETRequestDoc.Fields[0].Name = "matchers" + WEBSOCKETRequestDoc.Fields[0].Type = "[]matchers.Matcher" + WEBSOCKETRequestDoc.Fields[0].Note = "" + WEBSOCKETRequestDoc.Fields[0].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument." + WEBSOCKETRequestDoc.Fields[0].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify" + WEBSOCKETRequestDoc.Fields[1].Name = "extractors" + WEBSOCKETRequestDoc.Fields[1].Type = "[]extractors.Extractor" + WEBSOCKETRequestDoc.Fields[1].Note = "" + WEBSOCKETRequestDoc.Fields[1].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response." + WEBSOCKETRequestDoc.Fields[1].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify" + WEBSOCKETRequestDoc.Fields[2].Name = "matchers-condition" + WEBSOCKETRequestDoc.Fields[2].Type = "string" + WEBSOCKETRequestDoc.Fields[2].Note = "" + WEBSOCKETRequestDoc.Fields[2].Description = "MatchersCondition is the condition between the matchers. Default is OR." + WEBSOCKETRequestDoc.Fields[2].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR." + WEBSOCKETRequestDoc.Fields[2].Values = []string{ + "and", + "or", + } + WEBSOCKETRequestDoc.Fields[3].Name = "address" + WEBSOCKETRequestDoc.Fields[3].Type = "string" + WEBSOCKETRequestDoc.Fields[3].Note = "" + WEBSOCKETRequestDoc.Fields[3].Description = "Address contains address for the request" + WEBSOCKETRequestDoc.Fields[3].Comments[encoder.LineComment] = "Address contains address for the request" + WEBSOCKETRequestDoc.Fields[4].Name = "inputs" + WEBSOCKETRequestDoc.Fields[4].Type = "[]websocket.Input" + WEBSOCKETRequestDoc.Fields[4].Note = "" + WEBSOCKETRequestDoc.Fields[4].Description = "Inputs contains inputs for the websocket protocol" + WEBSOCKETRequestDoc.Fields[4].Comments[encoder.LineComment] = "Inputs contains inputs for the websocket protocol" + WEBSOCKETRequestDoc.Fields[5].Name = "headers" + WEBSOCKETRequestDoc.Fields[5].Type = "map[string]string" + WEBSOCKETRequestDoc.Fields[5].Note = "" + WEBSOCKETRequestDoc.Fields[5].Description = "Headers contains headers for the request." + WEBSOCKETRequestDoc.Fields[5].Comments[encoder.LineComment] = "Headers contains headers for the request." + WEBSOCKETRequestDoc.Fields[6].Name = "attack" + WEBSOCKETRequestDoc.Fields[6].Type = "generators.AttackTypeHolder" + WEBSOCKETRequestDoc.Fields[6].Note = "" + WEBSOCKETRequestDoc.Fields[6].Description = "Attack is the type of payload combinations to perform.\n\nSniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." + WEBSOCKETRequestDoc.Fields[6].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." + WEBSOCKETRequestDoc.Fields[6].Values = []string{ + "sniper", + "pitchfork", + "clusterbomb", + } + WEBSOCKETRequestDoc.Fields[7].Name = "payloads" + WEBSOCKETRequestDoc.Fields[7].Type = "map[string]interface{}" + WEBSOCKETRequestDoc.Fields[7].Note = "" + WEBSOCKETRequestDoc.Fields[7].Description = "Payloads contains any payloads for the current request.\n\nPayloads support both key-values combinations where a list\nof payloads is provided, or optionally a single file can also\nbe provided as payload which will be read on run-time." + WEBSOCKETRequestDoc.Fields[7].Comments[encoder.LineComment] = "Payloads contains any payloads for the current request." + + WEBSOCKETInputDoc.Type = "websocket.Input" + WEBSOCKETInputDoc.Comments[encoder.LineComment] = "" + WEBSOCKETInputDoc.Description = "" + WEBSOCKETInputDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "websocket.Request", + FieldName: "inputs", + }, + } + WEBSOCKETInputDoc.Fields = make([]encoder.Doc, 2) + WEBSOCKETInputDoc.Fields[0].Name = "data" + WEBSOCKETInputDoc.Fields[0].Type = "string" + WEBSOCKETInputDoc.Fields[0].Note = "" + WEBSOCKETInputDoc.Fields[0].Description = "Data is the data to send as the input.\n\nIt supports DSL Helper Functions as well as normal expressions." + WEBSOCKETInputDoc.Fields[0].Comments[encoder.LineComment] = "Data is the data to send as the input." + + WEBSOCKETInputDoc.Fields[0].AddExample("", "TEST") + + WEBSOCKETInputDoc.Fields[0].AddExample("", "hex_decode('50494e47')") + WEBSOCKETInputDoc.Fields[1].Name = "name" + WEBSOCKETInputDoc.Fields[1].Type = "string" + WEBSOCKETInputDoc.Fields[1].Note = "" + WEBSOCKETInputDoc.Fields[1].Description = "Name is the optional name of the data read to provide matching on." + WEBSOCKETInputDoc.Fields[1].Comments[encoder.LineComment] = "Name is the optional name of the data read to provide matching on." + + WEBSOCKETInputDoc.Fields[1].AddExample("", "prefix") + WORKFLOWSWorkflowTemplateDoc.Type = "workflows.WorkflowTemplate" WORKFLOWSWorkflowTemplateDoc.Comments[encoder.LineComment] = "" WORKFLOWSWorkflowTemplateDoc.Description = "" @@ -1144,12 +1315,16 @@ func GetTemplateDoc() *encoder.FileDoc { &HTTPRequestDoc, &MATCHERSMatcherDoc, &EXTRACTORSExtractorDoc, + &GENERATORSAttackTypeHolderDoc, &DNSRequestDoc, &FILERequestDoc, &NETWORKRequestDoc, &NETWORKInputDoc, &HEADLESSRequestDoc, &ENGINEActionDoc, + &SSLRequestDoc, + &WEBSOCKETRequestDoc, + &WEBSOCKETInputDoc, &WORKFLOWSWorkflowTemplateDoc, &WORKFLOWSMatcherDoc, }, From 7af5121de898dda227b4906445dcaa25b315a585 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Fri, 12 Nov 2021 19:06:59 +0530 Subject: [PATCH 120/146] Added high level nuclei architecture overview (#1177) * Added high level nuclei architecture overview --- DESIGN.md | 604 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 604 insertions(+) create mode 100644 DESIGN.md diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 00000000..f9875b0b --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,604 @@ +# Nuclei Architecture Document + +A brief overview of Nuclei Engine architecture. This document will be kept updated as the engine progresses. + +## pkg/templates + +### Template + +Template is the basic unit of input to the engine which describes the requests to be made, matching to be done, data to extract, etc. + +The template structure is described here. Template level attributes are defined here as well as convenience methods to validate, parse and compile templates creating executers. + +Any attributes etc required for the template, engine or requests to function are also set here. + +Workflows are also compiled, their templates are loaded and compiled as well. Any validations etc on the paths provided are also done here. + +`Parse` function is the main entry point which returns a template for a `filePath` and `executorOptions`. It compiles all the requests for the templates, all the workflows, as well as any self-contained request etc. It also caches the templates in an in-memory cache. + +### Preprocessors + +Preprocessors are also applied here which can do things at template level. They get data of the template which they can alter at will on runtime. This is used in the engine to do random string generation. + +Custom processor can be used if they satisfy the following interface. + +```go +type Preprocessor interface { + Process(data []byte) []byte +} +``` + +## pkg/model + +Model package implements Information structure for Nuclei Templates. `Info` contains all major metadata information for the template. `Classification` structure can also be used to provide additional context to vulnerability data. + +It also specifies a `WorkflowLoader` interface that is used during workflow loading in template compilation stage. + +```go +type WorkflowLoader interface { + GetTemplatePathsByTags(tags []string) []string + GetTemplatePaths(templatesList []string, noValidate bool) []string +} +``` + +## pkg/protocols + +Protocols package implements all the request protocols supported by Nuclei. This includes http, dns, network, headless and file requests as of now. + +### Request + +It exposes a `Request` interface that is implemented by all the request protocols supported. + +```go +// Request is an interface implemented any protocol based request generator. +type Request interface { + Compile(options *ExecuterOptions) error + Requests() int + GetID() string + Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) + Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} + ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error + MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent + MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent + GetCompiledOperators() []*operators.Operators +} +``` + +Many of these methods are similar across protocols while some are very protocol specific. + +A brief overview of the methods is provided below - + +- **Compile** - Compiles the request with provided options. +- **Requests** - Returns total requests made. +- **GetID** - Returns any ID for the request +- **Match** - Used to perform matching for patterns using matchers +- **Extract** - Used to perform extraction for patterns using extractors +- **ExecuteWithResults** - Request execution function for input. +- **MakeResultEventItem** - Creates a single result event for the intermediate `InternalWrappedEvent` output structure. +- **MakeResultEvent** - Returns a slice of results based on an `InternalWrappedEvent` internal output event. +- **GetCompiledOperators** - Returns the compiled operators for the request. + +`MakeDefaultResultEvent` function can be used as a default for `MakeResultEvent` function when no protocol-specific features need to be implemented for result generation. + +For reference protocol requests implementations, one can look at the below packages - + +1. [pkg/protocols/http](./v2/pkg/protocols/http) +2. [pkg/protocols/dns](./v2/pkg/protocols/dns) +3. [pkg/protocols/network](./v2/pkg/protocols/network) + +### Executer + +All these different requests interfaces are converted to an Executer which is also an interface defined in `pkg/protocols` which is used during final execution of the template. + +```go +// Executer is an interface implemented any protocol based request executer. +type Executer interface { + Compile() error + Requests() int + Execute(input string) (bool, error) + ExecuteWithResults(input string, callback OutputEventCallback) error +} +``` + +The `ExecuteWithResults` function accepts a callback, which gets provided with results during execution in form of `*output.InternalWrappedEvent` structure. + +The default executer is provided in `pkg/protocols/common/executer` . It takes a list of Requests and relevant `ExecuterOptions` and implements the Executer interface required for template execution. The executer during Template compilation process is created from this package and used as-is. + +A different executer is the Clustered Requests executer which implements the Nuclei Request clustering functionality in `pkg/protocols/common/clusterer` We have a single HTTP request in cases where multiple templates can be clustered and multiple operator lists to match/extract. The first HTTP request is executed while all the template matcher/extractor are evaluated separately. + +For Workflow execution, a separate RunWorkflow function is used which executes the workflow independently from the template execution. + +With this basic premise set, we can now start exploring the current runner implementation which will also walk us through the architecture of nuclei. + +## internal/runner + +### Template loading + +The first process after all CLI specific initialisation is the loading of template/workflow paths that the user wants to run. This is done by the packages described below. + +#### pkg/catalog + +This package is used to get paths using mixed syntax. It takes a template directory and performs resolving for template paths both from provided template directory as well as the current user directory. + +The syntax is very versatile and can include filenames, glob patterns, directories, absolute paths, and relative-paths. + + + +Next step is the initialisation of the reporting modules which is handled in `pkg/reporting`. + +#### pkg/reporting + +Reporting module contains exporters and trackers as well as a module for deduplication and a module for result formatting. + +Exporters and Trackers are interfaces defined in pkg/reporting. + +```go +// Tracker is an interface implemented by an issue tracker +type Tracker interface { + CreateIssue(event *output.ResultEvent) error +} + +// Exporter is an interface implemented by an issue exporter +type Exporter interface { + Close() error + Export(event *output.ResultEvent) error +} +``` + +Exporters include `Elasticsearch`, `markdown`, `sarif` . Trackers include `GitHub` , `Gitlab` and `Jira`. + +Each exporter and trackers implement their own configuration in YAML format and are very modular in nature, so adding new ones is easy. + + + +After reading all the inputs from various sources and initialisation other miscellaneous options, the next bit is the output writing which is done using `pkg/output` module. + +#### pkg/output + +Output package implements the output writing functionality for Nuclei. + +Output Writer implements the Writer interface which is called each time a result is found for nuclei. + +```go +// Writer is an interface which writes output to somewhere for nuclei events. +type Writer interface { + Close() + Colorizer() aurora.Aurora + Write(*ResultEvent) error + Request(templateID, url, requestType string, err error) +} +``` + +ResultEvent structure is passed to the Nuclei Output Writer which contains the entire detail of a found result. Various intermediary types like `InternalWrappedEvent` and `InternalEvent` are used throughout nuclei protocols and matchers to describe results in various stages of execution. + + + + Interactsh is also initialised if it is not explicitly disabled. + +#### pkg/protocols/common/interactsh + +Interactsh module is used to provide automatic Out of Band vulnerability identification in Nuclei. + +It uses two LRU caches, one for storing interactions for request URLs and one for storing requests for interaction URL. These both caches are used to correlated requests received to the Interactsh OOB server and Nuclei Instance. [Interactsh Client](https://github.com/projectdiscovery/interactsh/pkg/client) package does most of the heavy lifting of this module. + +Polling for interactions and server registration only starts when a template uses the interactsh module and is executed by nuclei. After that no registration is required for the entire run. + + + +### RunEnumeration + +Next we arrive in the `RunEnumeration` function of the runner. + +`HostErrorsCache` is initialised which is used throughout the run of Nuclei enumeration to keep track of errors per host and skip further requests if the errors are greater than the provided threshold. The functionality for the error tracking cache is defined in [hosterrorscache.go](https://github.com/projectdiscovery/nuclei/blob/master/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go) and is pretty simplistic in nature. + +Next the `WorkflowLoader` is initialised which used to load workflows. It exists in `v2/pkg/parsers/workflow_loader.go` + +The loader is initialised moving forward which is responsible for Using Catalog, Passed Tags, Filters, Paths, etc to return compiled `Templates` and `Workflows`. + +#### pkg/catalog/loader + +First the input passed by the user as paths is normalised to absolute paths which is done by the `pkg/catalog` module. Next the path filter module is used to removed the excluded template/workflows paths. + +`pkg/parsers` module's `LoadTemplate`,`LoadWorkflow` functions are used to check if the templates pass the validation + are not excluded via tags/severity/etc filters. If all checks are passed, then the template/workflow is parsed and returned in a compiled form by the `pkg/templates`'s `Parse` function. + +`Parse` function performs compilation of all the requests in a template + creates Executers from them returning a runnable Template/Workflow structure. + +Clustering module comes in next whose job is to cluster identical HTTP GET requests together (as a lot of the templates perform the same get requests many times, it's a good way to save many requests on large scans with lots of templates). + +### pkg/operators + +Operators package implements all of the matching and extracting logic of Nuclei. + +```go +// Operators contains the operators that can be applied on protocols +type Operators struct { + Matchers []*matchers.Matcher + Extractors []*extractors.Extractor + MatchersCondition string +} +``` + +A protocol only needs to embed the `operators.Operators` type shown above and it can utilise all the matching/extracting functionality of nuclei. + +```go +// MatchFunc performs matching operation for a matcher on model and returns true or false. +type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) + +// ExtractFunc performs extracting operation for an extractor on model and returns true or false. +type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} + +// Execute executes the operators on data and returns a result structure +func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc, isDebug bool) (*Result, bool) +``` + +The core of this process is the Execute function which takes an input dictionary as well as a Match and Extract function and return a `Result` structure which is used later during nuclei execution to check for results. + +```go +// Result is a result structure created from operators running on data. +type Result struct { + Matched bool + Extracted bool + Matches map[string][]string + Extracts map[string][]string + OutputExtracts []string + DynamicValues map[string]interface{} + PayloadValues map[string]interface{} +} +``` + +The internal logics for matching and extracting for things like words, regexes, jq, paths, etc is specified in `pkg/operators/matchers`, `pkg/operators/extractors`. Those packages should be investigated for further look into the topic. + + +### Template Execution + +`pkg/core` provides the engine mechanism which runs the templates/workflows on inputs. It exposes an `Execute` function which does the task of execution while also doing template clustring. The clustering can also be disbled optionally by the user. + +An example of using the core engine is provided below. + +```go +engine := core.New(r.options) +engine.SetExecuterOptions(executerOpts) +results := engine.ExecuteWithOpts(finalTemplates, r.hmapInputProvider, true) +``` + +### Using Nuclei From Go Code + +An example of using Nuclei From Go Code to run templates on targets is provided below. + +```go +package main + +import ( + "fmt" + "log" + "os" + "path" + + "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" + "github.com/projectdiscovery/nuclei/v2/pkg/core" + "github.com/projectdiscovery/nuclei/v2/pkg/core/inputs" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/parsers" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "go.uber.org/ratelimit" +) + +func main() { + cache := hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount) + defer cache.Close() + + mockProgress := &testutils.MockProgressClient{} + reportingClient, _ := reporting.New(&reporting.Options{}, "") + defer reportingClient.Close() + + outputWriter := testutils.NewMockOutputWriter() + outputWriter.WriteCallback = func(event *output.ResultEvent) { + fmt.Printf("Got Result: %v\n", event) + } + + defaultOpts := types.DefaultOptions() + protocolstate.Init(defaultOpts) + protocolinit.Init(defaultOpts) + + defaultOpts.Templates = goflags.StringSlice{"dns/cname-service-detection.yaml"} + defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags + + interactOpts := interactsh.NewDefaultOptions(outputWriter, reportingClient, mockProgress) + interactClient, err := interactsh.New(interactOpts) + if err != nil { + log.Fatalf("Could not create interact client: %s\n", err) + } + defer interactClient.Close() + + home, _ := os.UserHomeDir() + catalog := catalog.New(path.Join(home, "nuclei-templates")) + executerOpts := protocols.ExecuterOptions{ + Output: outputWriter, + Options: defaultOpts, + Progress: mockProgress, + Catalog: catalog, + IssuesClient: reportingClient, + RateLimiter: ratelimit.New(150), + Interactsh: interactClient, + HostErrorsCache: cache, + Colorizer: aurora.NewAurora(true), + } + engine := core.New(defaultOpts) + engine.SetExecuterOptions(executerOpts) + + workflowLoader, err := parsers.NewLoader(&executerOpts) + if err != nil { + log.Fatalf("Could not create workflow loader: %s\n", err) + } + executerOpts.WorkflowLoader = workflowLoader + + store, err := loader.New(loader.NewConfig(defaultOpts, catalog, executerOpts)) + if err != nil { + log.Fatalf("Could not create loader client: %s\n", err) + } + store.Load() + + input := &inputs.SimpleInputProvider{Inputs: []string{"docs.hackerone.com"}} + _ = engine.Execute(store.Templates(), input) +} +``` + +### Adding a New Protocol + +Protocols form the core of Nuclei Engine. All the request types like `http`, `dns`, etc are implemented in form of protocol requests. + +A protocol must implement the `Protocol` and `Request` interfaces described above in `pkg/protocols`. We'll take the example of an existing protocol implementation - websocket for this short reference around Nuclei internals. + +The code for the websocket protocol is contained in `pkg/protocols/others/websocket`. + +Below a high level skeleton of the websocket implementation is provided with all the important parts present. + +```go +package websocket + +// Request is a request for the Websocket protocol +type Request struct { + // Operators for the current request go here. + operators.Operators `yaml:",inline,omitempty"` + CompiledOperators *operators.Operators `yaml:"-"` + + // description: | + // Address contains address for the request + Address string `yaml:"address,omitempty" jsonschema:"title=address for the websocket request,description=Address contains address for the request"` + + // declarations here +} + +// Compile compiles the request generators preparing any requests possible. +func (r *Request) Compile(options *protocols.ExecuterOptions) error { + r.options = options + + // request compilation here as well as client creation + + if len(r.Matchers) > 0 || len(r.Extractors) > 0 { + compiled := &r.Operators + if err := compiled.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + r.CompiledOperators = compiled + } + return nil +} + +// Requests returns the total number of requests the rule will perform +func (r *Request) Requests() int { + if r.generator != nil { + return r.generator.NewIterator().Total() + } + return 1 +} + +// GetID returns the ID for the request if any. +func (r *Request) GetID() string { + return "" +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + // payloads init here + if err := r.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { + return err + } + return nil +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + header := http.Header{} + + // make the actual request here after setting all options + + event := eventcreator.CreateEventWithAdditionalOptions(r, data, r.options.Options.Debug || r.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + internalWrappedEvent.OperatorsResult.PayloadValues = payloadValues + }) + if r.options.Options.Debug || r.options.Options.DebugResponse { + responseOutput := responseBuilder.String() + gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", r.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, r.options.Options.NoColor)) + } + + callback(event) + return nil +} + +func (r *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ + TemplateID: types.ToString(r.options.TemplateID), + TemplatePath: types.ToString(r.options.TemplatePath), + // ... setting more values for result event + } + return data +} + +// Match performs matching operation for a matcher on model and returns: +// true and a list of matched snippets if the matcher type is supports it +// otherwise false and an empty string slice +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + return protocols.MakeDefaultMatchFunc(data, matcher) +} + +// Extract performs extracting operation for an extractor on model and returns true or false. +func (r *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { + return protocols.MakeDefaultExtractFunc(data, matcher) +} + +// MakeResultEvent creates a result event from internal wrapped event +func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(r, wrapped) +} + +// GetCompiledOperators returns a list of the compiled operators +func (r *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{r.CompiledOperators} +} + +// Type returns the type of the protocol request +func (r *Request) Type() templateTypes.ProtocolType { + return templateTypes.WebsocketProtocol +} +``` + +Almost all of these protocols have boilerplate functions for which default implementations have been provided in the `providers` package. Examples are the implementation of `Match`, `Extract`, `MakeResultEvent`, GetCompiledOperators`, etc which are almost same throughout Nuclei protocols code. It is enough to copy-paste them unless customization is required. + +`eventcreator` package offers `CreateEventWithAdditionalOptions` function which can be used to create result events after doing request execution. + +Step by step description of how to add a new protocol to Nuclei - + +1. Add the protocol implementation in `pkg/protocols` directory. If it's a small protocol with less number of options, considering adding it to the `pkg/protocols/others` directory. Add the enum for the new protocol to `v2/pkg/templates/types/types.go`. + +2. Add the protocol request structure to the `Template` structure fields. This is done in `pkg/templates/templates.go` with the corresponding import line. + +```go + +import ( + ... + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/websocket" +) + +// Template is a YAML input file which defines all the requests and +// other metadata for a template. +type Template struct { + ... + // description: | + // Websocket contains the Websocket request to make in the template. + RequestsWebsocket []*websocket.Request `yaml:"websocket,omitempty" json:"websocket,omitempty" jsonschema:"title=websocket requests to make,description=Websocket requests to make for the template"` + ... +} +``` + +Also add the protocol case to the `Type` function as well as the `TemplateTypes` array in the same `templates.go` file. + +```go +// TemplateTypes is a list of accepted template types +var TemplateTypes = []string{ + ... + "websocket", +} + +// Type returns the type of the template +func (t *Template) Type() templateTypes.ProtocolType { + ... + case len(t.RequestsWebsocket) > 0: + return templateTypes.WebsocketProtocol + default: + return "" + } +} +``` + +3. Add the protocol request to the `Requests` function and `compileProtocolRequests` function in the `compile.go` file in same directory. + +```go + +// Requests returns the total request count for the template +func (template *Template) Requests() int { + return len(template.RequestsDNS) + + ... + len(template.RequestsSSL) + + len(template.RequestsWebsocket) +} + + +// compileProtocolRequests compiles all the protocol requests for the template +func (template *Template) compileProtocolRequests(options protocols.ExecuterOptions) error { + ... + + case len(template.RequestsWebsocket) > 0: + requests = template.convertRequestToProtocolsRequest(template.RequestsWebsocket) + } + template.Executer = executer.NewExecuter(requests, &options) + return nil +} +``` + +That's it, you've added a new protocol to Nuclei. The next good step would be to write integration tests which are described in `integration-tests` and `cmd/integration-tests` directories. + +## Project Structure + +- [v2/pkg/reporting](./v2/pkg/reporting) - Reporting modules for nuclei. +- [v2/pkg/reporting/exporters/sarif](./v2/pkg/reporting/exporters/sarif) - Sarif Result Exporter +- [v2/pkg/reporting/exporters/markdown](./v2/pkg/reporting/exporters/markdown) - Markdown Result Exporter +- [v2/pkg/reporting/exporters/es](./v2/pkg/reporting/exporters/e) - Elasticsearch Result Exporter +- [v2/pkg/reporting/dedupe](./v2/pkg/reporting/dedupe) - Dedupe module for Results +- [v2/pkg/reporting/trackers/gitlab](./v2/pkg/reporting/trackers/gitlab) - Gitlab Issue Tracker Exporter +- [v2/pkg/reporting/trackers/jira](./v2/pkg/reporting/trackers/jira) - Jira Issue Tracker Exporter +- [v2/pkg/reporting/trackers/github](./v2/pkg/reporting/trackers/github) - Github Issue Tracker Exporter +- [v2/pkg/reporting/format](./v2/pkg/reporting/format) - Result Formatting Functions +- [v2/pkg/parsers](./v2/pkg/parsers) - Implements template as well as workflow loader for initial template discovery, validation and - loading. +- [v2/pkg/types](./v2/pkg/types) - Contains CLI options as well as misc helper functions. +- [v2/pkg/progress](./v2/pkg/progress) - Progress tracking +- [v2/pkg/operators](./v2/pkg/operators) - Operators for Nuclei +- [v2/pkg/operators/common/dsl](./v2/pkg/operators/common/dsl) - DSL functions for Nuclei YAML Syntax +- [v2/pkg/operators/matchers](./v2/pkg/operators/matchers) - Matchers implementation +- [v2/pkg/operators/extractors](./v2/pkg/operators/extractors) - Extractors implementation +- [v2/pkg/catalog](./v2/pkg/catalog) - Template loading from disk helpers +- [v2/pkg/catalog/config](./v2/pkg/catalog/config) - Internal configuration management +- [v2/pkg/catalog/loader](./v2/pkg/catalog/loader) - Implements loading and validation of templates and workflows. +- [v2/pkg/catalog/loader/filter](./v2/pkg/catalog/loader/filter) - Filter filters templates based on tags and paths +- [v2/pkg/output](./v2/pkg/output) - Output module for nuclei +- [v2/pkg/workflows](./v2/pkg/workflows) - Workflow execution logic + declarations +- [v2/pkg/utils](./v2/pkg/utils) - Utility functions +- [v2/pkg/model](./v2/pkg/model) - Template Info + misc +- [v2/pkg/templates](./v2/pkg/templates) - Templates core starting point +- [v2/pkg/templates/cache](./v2/pkg/templates/cache) - Templates cache +- [v2/pkg/protocols](./v2/pkg/protocol) - Protocol Specification +- [v2/pkg/protocols/file](./v2/pkg/protocols/file) - File protocol +- [v2/pkg/protocols/network](./v2/pkg/protocols/network) - Network protocol +- [v2/pkg/protocols/common/expressions](./v2/pkg/protocols/common/expressions) - Expression evaluation + Templating Support +- [v2/pkg/protocols/common/interactsh](./v2/pkg/protocols/common/interactsh) - Interactsh integration +- [v2/pkg/protocols/common/generators](./v2/pkg/protocols/common/generators) - Payload support for Requests (Sniper, etc) +- [v2/pkg/protocols/common/executer](./v2/pkg/protocols/common/executer) - Default Template Executer +- [v2/pkg/protocols/common/replacer](./v2/pkg/protocols/common/replacer) - Template replacement helpers +- [v2/pkg/protocols/common/clusterer](./v2/pkg/protocols/common/clusterer) - HTTP Request Clustering Implementation +- [v2/pkg/protocols/common/helpers/eventcreator](./v2/pkg/protocols/common/helpers/eventcreator) - Result event creator +- [v2/pkg/protocols/common/helpers/responsehighlighter](./v2/pkg/protocols/common/helpers/responsehighlighter) - Debug response highlighter +- [v2/pkg/protocols/common/helpers/deserialization](./v2/pkg/protocols/common/helpers/deserialization) - Deserialization helper functions +- [v2/pkg/protocols/common/hosterrorscache](./v2/pkg/protocols/common/hosterrorscache) - Host errors cache for tracking erroring hosts +- [v2/pkg/protocols/offlinehttp](./v2/pkg/protocols/offlinehttp) - Offline http protocol +- [v2/pkg/protocols/http](./v2/pkg/protocols/http) - HTTP protocol +- [v2/pkg/protocols/http/race](./v2/pkg/protocols/http/race) - HTTP Race Module +- [v2/pkg/protocols/http/raw](./v2/pkg/protocols/http/raw) - HTTP Raw Request Support +- [v2/pkg/protocols/headless](./v2/pkg/protocols/headless) - Headless Module +- [v2/pkg/protocols/headless/engine](./v2/pkg/protocols/headless/engine) - Internal Headless implementation +- [v2/pkg/protocols/dns](./v2/pkg/protocols/dns) - DNS protocol +- [v2/pkg/projectfile](./v2/pkg/projectfile) - Project File Implementation + +### Notes + +1. The matching as well as interim output functionality is a bit complex, we should simplify it a bit as well. From cc33366acb69c55c4fb7049a1867227fac376eb0 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 14 Nov 2021 15:30:28 +0530 Subject: [PATCH 121/146] fixed a conditon with workflow type switch --- v2/pkg/templates/templates.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 39e0e447..d34cd1ae 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -108,7 +108,7 @@ func (t *Template) Type() types.ProtocolType { return types.HeadlessProtocol case len(t.RequestsNetwork) > 0: return types.NetworkProtocol - case t.CompiledWorkflow != nil: + case len(t.Workflow.Workflows) > 0: return types.WorkflowProtocol case len(t.RequestsSSL) > 0: return types.SSLProtocol From 50a816fce9e28587b48b73b250e7aa6dbe02c14f Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 16 Nov 2021 20:02:39 +0530 Subject: [PATCH 122/146] Fixed a crash with uninitialized interactsh client (#1251) * Fixed a crash with uninitialized interactsh client --- v2/internal/runner/runner.go | 29 +++++++++-------- v2/pkg/operators/matchers/compile.go | 4 --- .../protocols/common/interactsh/interactsh.go | 6 +++- v2/pkg/protocols/http/build_request.go | 31 +++++++++++++------ v2/pkg/protocols/network/request.go | 7 ++++- .../protocols/offlinehttp/operators_test.go | 1 + 6 files changed, 48 insertions(+), 30 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 9a5981fa..5e71b76f 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -144,22 +144,21 @@ func New(options *types.Options) (*Runner, error) { } } - if !options.NoInteractsh { - opts := interactsh.NewDefaultOptions(runner.output, runner.issuesClient, runner.progress) - opts.Debug = runner.options.Debug - opts.ServerURL = options.InteractshURL - opts.Authorization = options.InteractshToken - opts.CacheSize = int64(options.InteractionsCacheSize) - opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second - opts.ColldownPeriod = time.Duration(options.InteractionsCooldownPeriod) * time.Second - opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second + opts := interactsh.NewDefaultOptions(runner.output, runner.issuesClient, runner.progress) + opts.Debug = runner.options.Debug + opts.ServerURL = options.InteractshURL + opts.Authorization = options.InteractshToken + opts.CacheSize = int64(options.InteractionsCacheSize) + opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second + opts.ColldownPeriod = time.Duration(options.InteractionsCooldownPeriod) * time.Second + opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second + opts.NoInteractsh = runner.options.NoInteractsh - interactshClient, err := interactsh.New(opts) - if err != nil { - gologger.Error().Msgf("Could not create interactsh client: %s", err) - } else { - runner.interactsh = interactshClient - } + interactshClient, err := interactsh.New(opts) + if err != nil { + gologger.Error().Msgf("Could not create interactsh client: %s", err) + } else { + runner.interactsh = interactshClient } if options.RateLimitMinute > 0 { diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index 7c7c5555..d4937614 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -29,10 +29,6 @@ func (m *Matcher) CompileMatchers() error { if !ok { return fmt.Errorf("unknown matcher type specified: %s", m.Type) } - // By default, match on body if user hasn't provided any specific items - if m.Part == "" { - m.Part = "response" - } // Compile the regexes for _, regex := range m.Regex { diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index c11f5ad1..7c3e02f8 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -72,6 +72,8 @@ type Options struct { Progress progress.Progress // Debug specifies whether debugging output should be shown for interactsh-client Debug bool + + NoInteractsh bool } const defaultMaxInteractionsCount = 5000 @@ -118,6 +120,9 @@ func NewDefaultOptions(output output.Writer, reporting *reporting.Client, progre } func (c *Client) firstTimeInitializeClient() error { + if c.options.NoInteractsh { + return nil // do not init if disabled + } interactsh, err := client.New(&client.Options{ ServerURL: c.options.ServerURL, Token: c.options.Authorization, @@ -221,7 +226,6 @@ func (c *Client) Close() bool { // It accepts data to replace as well as the URL to replace placeholders // with generated uniquely for each request. func (c *Client) ReplaceMarkers(data string, interactshURLs []string) (string, []string) { - for strings.Contains(data, interactshURLMarker) { url := c.URL() interactshURLs = append(interactshURLs, url) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 4b4ac9f2..6b9b01f1 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -64,10 +64,15 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa } ctx := context.Background() - data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) - - for payloadName, payloadValue := range payloads { - payloads[payloadName], r.interactshURLs = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), r.interactshURLs) + if r.options.Interactsh != nil { + data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) + for payloadName, payloadValue := range payloads { + payloads[payloadName], r.interactshURLs = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), r.interactshURLs) + } + } else { + for payloadName, payloadValue := range payloads { + payloads[payloadName] = types.ToString(payloadValue) + } } parsed, err := url.Parse(baseURL) @@ -171,7 +176,9 @@ func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) { // MakeHTTPRequestFromModel creates a *http.Request from a request template func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values, generatorValues map[string]interface{}) (*generatedRequest, error) { - data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) + if r.options.Interactsh != nil { + data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) + } // Combine the template payloads along with base // request values. @@ -204,7 +211,9 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st // makeHTTPRequestFromRaw creates a *http.Request from a raw request func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}) (*generatedRequest, error) { - data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) + if r.options.Interactsh != nil { + data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) + } return r.handleRawWithPayloads(ctx, data, baseURL, values, payloads) } @@ -268,7 +277,9 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest func (r *requestGenerator) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) { // Set the header values requested for header, value := range r.request.Headers { - value, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(value, r.interactshURLs) + if r.options.Interactsh != nil { + value, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(value, r.interactshURLs) + } value, err := expressions.Evaluate(value, values) if err != nil { return nil, errors.Wrap(err, "could not evaluate helper expressions") @@ -286,8 +297,10 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte // Check if the user requested a request body if r.request.Body != "" { - var body string - body, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(r.request.Body, r.interactshURLs) + body := r.request.Body + if r.options.Interactsh != nil { + body, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(r.request.Body, r.interactshURLs) + } body, err := expressions.Evaluate(body, values) if err != nil { return nil, errors.Wrap(err, "could not evaluate helper expressions") diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 98c65ee4..f5229d12 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -140,7 +140,6 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input case "hex": data, err = hex.DecodeString(input.Data) default: - input.Data, interactshURLs = request.options.Interactsh.ReplaceMarkers(input.Data, []string{}) data = []byte(input.Data) } if err != nil { @@ -150,6 +149,12 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input } reqBuilder.Grow(len(input.Data)) + if request.options.Interactsh != nil { + var transformedData string + transformedData, interactshURLs = request.options.Interactsh.ReplaceMarkers(string(data), []string{}) + data = []byte(transformedData) + } + finalData, dataErr := expressions.EvaluateByte(data, payloads) if dataErr != nil { request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), dataErr) diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index 8669d8b5..05fe9468 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -154,6 +154,7 @@ func TestHTTPOperatorExtract(t *testing.T) { extractor := &extractors.Extractor{ Type: "kval", KVal: []string{"test-header"}, + Part: "header", } err = extractor.CompileExtractors() require.Nil(t, err, "could not compile kval extractor") From eaa0d370656ff6bb1a7e40556245660d9f895fd4 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Wed, 17 Nov 2021 02:04:27 +0530 Subject: [PATCH 123/146] feat #1092: Validate binary values + precompile them as well (#1213) * Added validation for binary values + precompile them * Changed name of the binary matcher field Co-authored-by: sandeep --- v2/pkg/operators/matchers/compile.go | 9 +++++++++ v2/pkg/operators/matchers/match.go | 19 ++++--------------- v2/pkg/operators/matchers/matchers.go | 1 + 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index d4937614..0efebb1a 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -39,6 +39,15 @@ func (m *Matcher) CompileMatchers() error { m.regexCompiled = append(m.regexCompiled, compiled) } + // Compile and validate binary Values in matcher + for _, value := range m.Binary { + if decoded, err := hex.DecodeString(value); err != nil { + return fmt.Errorf("could not hex decode binary: %s", value) + } else { + m.binaryDecoded = append(m.binaryDecoded, string(decoded)) + } + } + // Compile the dsl expressions for _, expr := range m.DSL { compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index df71a144..84601be6 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -1,10 +1,8 @@ package matchers import ( - "encoding/hex" "strings" - "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" ) @@ -120,17 +118,8 @@ func (m *Matcher) MatchRegex(corpus string) (bool, []string) { func (m *Matcher) MatchBinary(corpus string) (bool, []string) { var matchedBinary []string // Iterate over all the words accepted as valid - for i, binary := range m.Binary { - // Continue if the word doesn't match - hexa, err := hex.DecodeString(binary) - if err != nil { - gologger.Warning().Msgf("Could not hex encode the given binary matcher value: '%s'", binary) - if m.condition == ANDCondition { - return false, []string{} - } - continue - } - if !strings.Contains(corpus, string(hexa)) { + for i, binary := range m.binaryDecoded { + if !strings.Contains(corpus, binary) { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. if m.condition == ANDCondition { @@ -142,10 +131,10 @@ func (m *Matcher) MatchBinary(corpus string) (bool, []string) { // If the condition was an OR, return on the first match. if m.condition == ORCondition { - return true, []string{string(hexa)} + return true, []string{binary} } - matchedBinary = append(matchedBinary, string(hexa)) + matchedBinary = append(matchedBinary, binary) // If we are at the end of the words, return with true if len(m.Binary)-1 == i { diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index 594a6fc1..4c0fd1fe 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -115,6 +115,7 @@ type Matcher struct { // cached data for the compiled matcher condition ConditionType matcherType MatcherType + binaryDecoded []string regexCompiled []*regexp.Regexp dslCompiled []*govaluate.EvaluableExpression } From 4ff90c1373ca81f8800decd2086504b7e49a8e9e Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 18 Nov 2021 07:47:34 -0600 Subject: [PATCH 124/146] struct to enum changes for Dns Type (#1245) * struct to enum changes for Dns Type --- v2/pkg/protocols/dns/dns.go | 4 +- v2/pkg/protocols/dns/dns_test.go | 2 +- v2/pkg/protocols/dns/dns_types.go | 110 +++++++++++++++++++++ v2/pkg/protocols/dns/operators_test.go | 8 +- v2/pkg/protocols/dns/request_test.go | 2 +- v2/pkg/templates/templates_doc_examples.go | 2 +- 6 files changed, 119 insertions(+), 9 deletions(-) create mode 100644 v2/pkg/protocols/dns/dns_types.go diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index a42df7ef..a630762f 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -43,7 +43,7 @@ type Request struct { // - "MX" // - "TXT" // - "AAAA" - RequestType string `yaml:"type,omitempty" jsonschema:"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"` + RequestType DNSRequestTypeHolder `yaml:"type,omitempty" jsonschema:"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"` // description: | // Class is the class of the DNS request. // @@ -111,7 +111,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } request.class = classToInt(request.Class) request.options = options - request.question = questionTypeToInt(request.RequestType) + request.question = questionTypeToInt(request.RequestType.String()) return nil } diff --git a/v2/pkg/protocols/dns/dns_test.go b/v2/pkg/protocols/dns/dns_test.go index b0f8dddf..f44ba220 100644 --- a/v2/pkg/protocols/dns/dns_test.go +++ b/v2/pkg/protocols/dns/dns_test.go @@ -27,7 +27,7 @@ func TestDNSCompileMake(t *testing.T) { testutils.Init(options) const templateID = "testing-dns" request := &Request{ - RequestType: "A", + RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", Retries: 5, ID: templateID, diff --git a/v2/pkg/protocols/dns/dns_types.go b/v2/pkg/protocols/dns/dns_types.go new file mode 100644 index 00000000..e1354fed --- /dev/null +++ b/v2/pkg/protocols/dns/dns_types.go @@ -0,0 +1,110 @@ +package dns + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/alecthomas/jsonschema" +) + +// DNSRequestType is the type of the method specified +type DNSRequestType int + +const ( + A DNSRequestType = iota + 1 + NS + DS + CNAME + SOA + PTR + MX + TXT + AAAA + //limit + limit +) + +// DNSRequestTypeMapping is a table for conversion of method from string. +var DNSRequestTypeMapping = map[DNSRequestType]string{ + A: "A", + NS: "NS", + DS: "DS", + CNAME: "CNAME", + SOA: "SOA", + PTR: "PTR", + MX: "MX", + TXT: "TXT", + AAAA: "AAAA", +} + +// GetSupportedDNSRequestTypes returns list of supported types +func GetSupportedDNSRequestTypes() []DNSRequestType { + var result []DNSRequestType + for index := DNSRequestType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toDNSRequestTypes(valueToMap string) (DNSRequestType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range DNSRequestTypeMapping { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("Invalid DNS request type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToUpper(value)) +} + +func (t DNSRequestType) String() string { + return DNSRequestTypeMapping[t] +} + +// DNSRequestTypeHolder is used to hold internal type of the DNS type +type DNSRequestTypeHolder struct { + DNSRequestType DNSRequestType +} + +func (holder DNSRequestTypeHolder) String() string { + return holder.DNSRequestType.String() +} + +func (holder DNSRequestTypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "type of DNS request to make", + Description: "Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA", + } + for _, types := range GetSupportedDNSRequestTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *DNSRequestTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toDNSRequestTypes(marshalledTypes) + if err != nil { + return err + } + + holder.DNSRequestType = computedType + return nil +} + +func (holder *DNSRequestTypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.DNSRequestType.String()) +} + +func (holder DNSRequestTypeHolder) MarshalYAML() (interface{}, error) { + return holder.DNSRequestType.String(), nil +} diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index e5618f61..bb87b52e 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -23,7 +23,7 @@ func TestResponseToDSLMap(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - RequestType: "A", + RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", Retries: 5, ID: templateID, @@ -55,7 +55,7 @@ func TestDNSOperatorMatch(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - RequestType: "A", + RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", Retries: 5, ID: templateID, @@ -166,7 +166,7 @@ func TestDNSOperatorExtract(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - RequestType: "A", + RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", Retries: 5, ID: templateID, @@ -223,7 +223,7 @@ func TestDNSMakeResult(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - RequestType: "A", + RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", Retries: 5, ID: templateID, diff --git a/v2/pkg/protocols/dns/request_test.go b/v2/pkg/protocols/dns/request_test.go index d2ed8399..38827f81 100644 --- a/v2/pkg/protocols/dns/request_test.go +++ b/v2/pkg/protocols/dns/request_test.go @@ -20,7 +20,7 @@ func TestDNSExecuteWithResults(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - RequestType: "A", + RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", Retries: 5, ID: templateID, diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index b755de29..e6655c6d 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -38,7 +38,7 @@ var ( exampleNormalDNSRequest = &dns.Request{ Name: "{{FQDN}}", - RequestType: "CNAME", + RequestType: dns.DNSRequestTypeHolder{DNSRequestType: dns.CNAME}, Class: "inet", Retries: 2, Recursion: true, From 64a93a457051df381475ca9c00a3a508bbc55f68 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 18 Nov 2021 07:48:47 -0600 Subject: [PATCH 125/146] enum changes for network type (#1233) * enum changes for network type --- v2/pkg/protocols/network/network.go | 6 +- .../protocols/network/network_input_types.go | 100 ++++++++++++++++++ v2/pkg/protocols/network/request.go | 4 +- v2/pkg/protocols/network/request_test.go | 2 +- 4 files changed, 106 insertions(+), 6 deletions(-) create mode 100644 v2/pkg/protocols/network/network_input_types.go diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 83c9ac81..98037eed 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -105,7 +105,7 @@ type Input struct { // values: // - "hex" // - "text" - Type string `yaml:"type,omitempty" jsonschema:"title=type is the type of input data,description=Type of input specified in data field,enum=hex,enum=text"` + Type NetworkInputTypeHolder `yaml:"type,omitempty" jsonschema:"title=type is the type of input data,description=Type of input specified in data field,enum=hex,enum=text"` // description: | // Read is the number of bytes to read from socket. // @@ -153,7 +153,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } // Pre-compile any input dsl functions before executing the request. for _, input := range request.Inputs { - if input.Type != "" { + if input.Type.String() != "" { continue } if compiled, evalErr := expressions.Evaluate(input.Data, map[string]interface{}{}); evalErr == nil { @@ -167,7 +167,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { // check if inputs contains the payload var hasPayloadName bool for _, input := range request.Inputs { - if input.Type != "" { + if input.Type.String() != "" { continue } if expressions.ContainsVariablesWithNames(input.Data, map[string]interface{}{name: payload}) == nil { diff --git a/v2/pkg/protocols/network/network_input_types.go b/v2/pkg/protocols/network/network_input_types.go new file mode 100644 index 00000000..f96c3e49 --- /dev/null +++ b/v2/pkg/protocols/network/network_input_types.go @@ -0,0 +1,100 @@ +package network + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/alecthomas/jsonschema" +) + +// NetworkInputType is the type of the method specified +type NetworkInputType int + +const ( + hexType NetworkInputType = iota + 1 + textType + //limit + limit +) + +// NetworkInputMapping is a table for conversion of method from string. +var NetworkInputMapping = map[NetworkInputType]string{ + hexType: "hex", + textType: "text", +} + +// GetSupportedNetworkInputTypes returns list of supported types +func GetSupportedNetworkInputTypes() []NetworkInputType { + var result []NetworkInputType + for index := NetworkInputType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toNetworkInputTypes(valueToMap string) (NetworkInputType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range NetworkInputMapping { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("Invalid network type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToLower(value)) +} + +func (t NetworkInputType) String() string { + return NetworkInputMapping[t] +} + +// NetworkInputTypeHolder is used to hold internal type of the Network type +type NetworkInputTypeHolder struct { + NetworkInputType NetworkInputType +} + +func (holder NetworkInputTypeHolder) GetType() NetworkInputType { + return holder.NetworkInputType +} + +func (holder NetworkInputTypeHolder) String() string { + return holder.NetworkInputType.String() +} + +func (holder NetworkInputTypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "type is the type of input data", + Description: "description=Type of input specified in data field,enum=hex,enum=text", + } + for _, types := range GetSupportedNetworkInputTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *NetworkInputTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toNetworkInputTypes(marshalledTypes) + if err != nil { + return err + } + + holder.NetworkInputType = computedType + return nil +} + +func (holder *NetworkInputTypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.NetworkInputType.String()) +} + +func (holder NetworkInputTypeHolder) MarshalYAML() (interface{}, error) { + return holder.NetworkInputType.String(), nil +} diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index f5229d12..ae63dc0d 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -136,8 +136,8 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input for _, input := range request.Inputs { var data []byte - switch input.Type { - case "hex": + switch input.Type.GetType() { + case hexType: data, err = hex.DecodeString(input.Data) default: data = []byte(input.Data) diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index 8b44091e..0e8d194b 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -91,7 +91,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { require.Equal(t, "

Example Domain

", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results") finalEvent = nil - request.Inputs[0].Type = "hex" + request.Inputs[0].Type = NetworkInputTypeHolder{NetworkInputType: hexType} request.Inputs[0].Data = hex.EncodeToString([]byte(fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", parsed.Host))) t.Run("hex-to-string", func(t *testing.T) { From 25189c3ae79273b4c9ccc328fdee6c41704dc8c4 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 18 Nov 2021 13:50:07 +0000 Subject: [PATCH 126/146] Auto Generate Syntax Docs + JSONSchema [Thu Nov 18 13:50:07 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 4 +-- nuclei-jsonschema.json | 46 +++++++++++++++++++------------ v2/pkg/templates/templates_doc.go | 4 +-- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index d87ac890..8ea79829 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -2043,7 +2043,7 @@ name: '{{FQDN}}'
-type string +type DNSRequestTypeHolder
@@ -2609,7 +2609,7 @@ data: hex_decode('50494e47')
-type string +type NetworkInputTypeHolder
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 517876ea..da66f3e9 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -318,6 +318,22 @@ "title": "type of the attack", "description": "Type of the attack" }, + "dns.DNSRequestTypeHolder": { + "enum": [ + "A", + "NS", + "DS", + "CNAME", + "SOA", + "PTR", + "MX", + "TXT", + "AAAA" + ], + "type": "string", + "title": "type of DNS request to make", + "description": "Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA" + }, "dns.Request": { "properties": { "matchers": { @@ -356,18 +372,8 @@ "description": "Name is the Hostname to make DNS request for" }, "type": { - "enum": [ - "A", - "NS", - "DS", - "CNAME", - "SOA", - "PTR", - "MX", - "TXT", - "AAAA" - ], - "type": "string", + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/dns.DNSRequestTypeHolder", "title": "type of dns request to make", "description": "Type is the type of DNS request to make" }, @@ -754,11 +760,8 @@ "description": "Data is the data to send as the input" }, "type": { - "enum": [ - "hex", - "text" - ], - "type": "string", + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/network.NetworkInputTypeHolder", "title": "type is the type of input data", "description": "Type of input specified in data field" }, @@ -776,6 +779,15 @@ "additionalProperties": false, "type": "object" }, + "network.NetworkInputTypeHolder": { + "enum": [ + "hex", + "text" + ], + "type": "string", + "title": "type is the type of input data", + "description": "description=Type of input specified in data field,enum=hex,enum=text" + }, "network.Request": { "properties": { "id": { diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index b24ccdaa..c7f1b016 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -790,7 +790,7 @@ func init() { DNSRequestDoc.Fields[4].AddExample("", "{{FQDN}}") DNSRequestDoc.Fields[5].Name = "type" - DNSRequestDoc.Fields[5].Type = "string" + DNSRequestDoc.Fields[5].Type = "DNSRequestTypeHolder" DNSRequestDoc.Fields[5].Note = "" DNSRequestDoc.Fields[5].Description = "RequestType is the type of DNS request to make." DNSRequestDoc.Fields[5].Comments[encoder.LineComment] = "RequestType is the type of DNS request to make." @@ -997,7 +997,7 @@ func init() { NETWORKInputDoc.Fields[0].AddExample("", "hex_decode('50494e47')") NETWORKInputDoc.Fields[1].Name = "type" - NETWORKInputDoc.Fields[1].Type = "string" + NETWORKInputDoc.Fields[1].Type = "NetworkInputTypeHolder" NETWORKInputDoc.Fields[1].Note = "" NETWORKInputDoc.Fields[1].Description = "Type is the type of input specified in `data` field.\n\nDefault value is text, but hex can be used for hex formatted data." NETWORKInputDoc.Fields[1].Comments[encoder.LineComment] = "Type is the type of input specified in `data` field." From d078b7238102f77c3ec296e8adc6a6027c58b476 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 18 Nov 2021 07:50:21 -0600 Subject: [PATCH 127/146] enum changes for http method (#1234) * enum changes for http method --- SYNTAX-REFERENCE.md | 2 +- nuclei-jsonschema.json | 32 +++--- v2/pkg/protocols/http/build_request.go | 2 +- v2/pkg/protocols/http/build_request_test.go | 6 +- v2/pkg/protocols/http/cluster_test.go | 4 +- v2/pkg/protocols/http/http.go | 4 +- v2/pkg/protocols/http/http_method_types.go | 112 ++++++++++++++++++++ v2/pkg/protocols/http/operators_test.go | 8 +- v2/pkg/templates/templates_doc.go | 2 +- v2/pkg/templates/templates_doc_examples.go | 2 +- 10 files changed, 146 insertions(+), 28 deletions(-) create mode 100644 v2/pkg/protocols/http/http_method_types.go diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 8ea79829..ed19bd9c 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -880,7 +880,7 @@ Valid values:
-method string +method HTTPMethodTypeHolder
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index da66f3e9..1d179d68 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -576,6 +576,23 @@ "additionalProperties": false, "type": "object" }, + "http.HTTPMethodTypeHolder": { + "enum": [ + "GET", + "HEAD", + "POST", + "PUT", + "DELETE", + "CONNECT", + "OPTIONS", + "TRACE", + "PATCH", + "PURGE" + ], + "type": "string", + "title": "method is the http request method", + "description": "Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE" + }, "http.Request": { "properties": { "matchers": { @@ -637,19 +654,8 @@ "description": "Attack is the type of payload combinations to perform" }, "method": { - "enum": [ - "GET", - "HEAD", - "POST", - "PUT", - "DELETE", - "CONNECT", - "OPTIONS", - "TRACE", - "PATCH", - "PURGE" - ], - "type": "string", + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/http.HTTPMethodTypeHolder", "title": "method is the http request method", "description": "Method is the HTTP Request Method" }, diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 6b9b01f1..8c8a2dbc 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -191,7 +191,7 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st return nil, errors.Wrap(err, "could not evaluate helper expressions") } - method, err := expressions.Evaluate(r.request.Method, finalValues) + method, err := expressions.Evaluate(r.request.Method.String(), finalValues) if err != nil { return nil, errors.Wrap(err, "could not evaluate helper expressions") } diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index 429676a3..a77e3c94 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -71,7 +71,7 @@ func TestMakeRequestFromModal(t *testing.T) { ID: templateID, Name: "testing", Path: []string{"{{BaseURL}}/login.php"}, - Method: "POST", + Method: HTTPMethodTypeHolder{MethodType: HTTPPost}, Body: "username=test&password=pass", Headers: map[string]string{ "Content-Type": "application/x-www-form-urlencoded", @@ -103,7 +103,7 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) { ID: templateID, Name: "testing", Path: []string{"{{BaseURL}}?query=example"}, - Method: "GET", + Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -211,7 +211,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { ID: templateID, Name: "testing", Path: []string{"{{BaseURL}}/?u=http://{{interactsh-url}}/&href=http://{{interactsh-url}}/&action=http://{{interactsh-url}}/&host={{interactsh-url}}"}, - Method: "GET", + Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, diff --git a/v2/pkg/protocols/http/cluster_test.go b/v2/pkg/protocols/http/cluster_test.go index 136b3feb..a16db41e 100644 --- a/v2/pkg/protocols/http/cluster_test.go +++ b/v2/pkg/protocols/http/cluster_test.go @@ -10,6 +10,6 @@ func TestCanCluster(t *testing.T) { req := &Request{Unsafe: true} require.False(t, req.CanCluster(&Request{}), "could cluster unsafe request") - req = &Request{Path: []string{"{{BaseURL}}"}, Method: "GET"} - require.True(t, req.CanCluster(&Request{Path: []string{"{{BaseURL}}"}, Method: "GET"}), "could not cluster GET request") + req = &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}} + require.True(t, req.CanCluster(&Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}), "could not cluster GET request") } diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index f85e49c2..3fbe3a8e 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -66,7 +66,7 @@ type Request struct { // - "TRACE" // - "PATCH" // - "PURGE" - Method string `yaml:"method,omitempty" jsonschema:"title=method is the http request method,description=Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE"` + Method HTTPMethodTypeHolder `yaml:"method,omitempty" jsonschema:"title=method is the http request method,description=Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE"` // description: | // Body is an optional parameter which contains HTTP Request body. // examples: @@ -242,7 +242,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { var hasPayloadName bool // search for markers in all request parts var inputs []string - inputs = append(inputs, request.Method, request.Body) + inputs = append(inputs, request.Method.String(), request.Body) inputs = append(inputs, request.Raw...) for k, v := range request.customHeaders { inputs = append(inputs, fmt.Sprintf("%s: %s", k, v)) diff --git a/v2/pkg/protocols/http/http_method_types.go b/v2/pkg/protocols/http/http_method_types.go new file mode 100644 index 00000000..987dde97 --- /dev/null +++ b/v2/pkg/protocols/http/http_method_types.go @@ -0,0 +1,112 @@ +package http + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/alecthomas/jsonschema" +) + +// HTTPMethodType is the type of the method specified +type HTTPMethodType int + +const ( + HTTPGet HTTPMethodType = iota + 1 + HTTPHead + HTTPPost + HTTPPut + HTTPDelete + HTTPConnect + HTTPOptions + HTTPTrace + HTTPPatch + HTTPPurge + //limit + limit +) + +// HTTPMethodMapping is a table for conversion of method from string. +var HTTPMethodMapping = map[HTTPMethodType]string{ + HTTPGet: "GET", + HTTPHead: "HEAD", + HTTPPost: "POST", + HTTPPut: "PUT", + HTTPDelete: "DELETE", + HTTPConnect: "CONNECT", + HTTPOptions: "OPTIONS", + HTTPTrace: "TRACE", + HTTPPatch: "PATCH", + HTTPPurge: "PURGE", +} + +// GetSupportedHTTPMethodTypes returns list of supported types +func GetSupportedHTTPMethodTypes() []HTTPMethodType { + var result []HTTPMethodType + for index := HTTPMethodType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toHTTPMethodTypes(valueToMap string) (HTTPMethodType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range HTTPMethodMapping { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("Invalid HTTP method verb: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToUpper(value)) +} + +func (t HTTPMethodType) String() string { + return HTTPMethodMapping[t] +} + +// HTTPMethodTypeHolder is used to hold internal type of the HTTP Method +type HTTPMethodTypeHolder struct { + MethodType HTTPMethodType +} + +func (holder HTTPMethodTypeHolder) String() string { + return holder.MethodType.String() +} + +func (holder HTTPMethodTypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "method is the HTTP request method", + Description: "Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE", + } + for _, types := range GetSupportedHTTPMethodTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *HTTPMethodTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toHTTPMethodTypes(marshalledTypes) + if err != nil { + return err + } + + holder.MethodType = computedType + return nil +} + +func (holder *HTTPMethodTypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.MethodType.String()) +} + +func (holder HTTPMethodTypeHolder) MarshalYAML() (interface{}, error) { + return holder.MethodType.String(), nil +} diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index bc7b888a..7950bc64 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -25,7 +25,7 @@ func TestResponseToDSLMap(t *testing.T) { ID: templateID, Name: "testing", Path: []string{"{{BaseURL}}?test=1"}, - Method: "GET", + Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -55,7 +55,7 @@ func TestHTTPOperatorMatch(t *testing.T) { ID: templateID, Name: "testing", Path: []string{"{{BaseURL}}?test=1"}, - Method: "GET", + Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -143,7 +143,7 @@ func TestHTTPOperatorExtract(t *testing.T) { ID: templateID, Name: "testing", Path: []string{"{{BaseURL}}?test=1"}, - Method: "GET", + Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -257,7 +257,7 @@ func TestHTTPMakeResult(t *testing.T) { ID: templateID, Name: "testing", Path: []string{"{{BaseURL}}?test=1"}, - Method: "GET", + Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, Operators: operators.Operators{ Matchers: []*matchers.Matcher{{ Name: "test", diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index c7f1b016..52c5da75 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -344,7 +344,7 @@ func init() { "clusterbomb", } HTTPRequestDoc.Fields[8].Name = "method" - HTTPRequestDoc.Fields[8].Type = "string" + HTTPRequestDoc.Fields[8].Type = "HTTPMethodTypeHolder" HTTPRequestDoc.Fields[8].Note = "" HTTPRequestDoc.Fields[8].Description = "Method is the HTTP Request Method." HTTPRequestDoc.Fields[8].Comments[encoder.LineComment] = "Method is the HTTP Request Method." diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index e6655c6d..88d5e5e8 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -24,7 +24,7 @@ var ( Tags: stringslice.StringSlice{Value: "cve,cve2021,rce,ruby"}, } exampleNormalHTTPRequest = &http.Request{ - Method: "GET", + Method: http.HTTPMethodTypeHolder{MethodType: http.HTTPGet}, Path: []string{"{{BaseURL}}/.git/config"}, Operators: operators.Operators{ MatchersCondition: "and", From bef7bfc393fc27f16b275a8ba1e177868410ad5a Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 18 Nov 2021 13:51:52 +0000 Subject: [PATCH 128/146] Auto Generate Syntax Docs + JSONSchema [Thu Nov 18 13:51:52 UTC 2021] :robot: --- nuclei-jsonschema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 1d179d68..9ace729a 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -590,7 +590,7 @@ "PURGE" ], "type": "string", - "title": "method is the http request method", + "title": "method is the HTTP request method", "description": "Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE" }, "http.Request": { From 7f5f791e2342e5d4206d611be6d97f96f863235d Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 18 Nov 2021 14:52:11 +0100 Subject: [PATCH 129/146] Adding dns trace support in dns templates (#1236) * Adding dns trace support in dns templates + minor refactoring --- v2/go.mod | 2 +- v2/go.sum | 4 +- v2/pkg/protocols/common/executer/executer.go | 2 - .../protocols/common/expressions/variables.go | 80 ++++++++++--------- v2/pkg/protocols/dns/dns.go | 34 +++++++- v2/pkg/protocols/dns/operators.go | 19 ++++- v2/pkg/protocols/dns/operators_test.go | 12 +-- v2/pkg/protocols/dns/request.go | 41 +++++++++- v2/pkg/protocols/http/http.go | 2 +- v2/pkg/protocols/network/network.go | 2 +- 10 files changed, 145 insertions(+), 53 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 15980fdd..c7d51d01 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -36,7 +36,7 @@ require ( github.com/projectdiscovery/interactsh v0.0.6 github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df github.com/projectdiscovery/rawhttp v0.0.7 - github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a + github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660 github.com/projectdiscovery/retryablehttp-go v1.0.2 github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/yamldoc-go v1.0.2 diff --git a/v2/go.sum b/v2/go.sum index afea1cbb..c649ceb4 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -582,6 +582,7 @@ github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/d github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ= github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= @@ -618,8 +619,9 @@ github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohT github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= -github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a h1:WJQjr9qi/VjWhdNiGyNqcFi0967Gp0W3I769bCpHOJE= github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI= +github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660 h1:Ooa5htghPkdyfpzy6Y5KLdyv4w8ePZWmfzFSPQlJStQ= +github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660/go.mod h1:UfszkO3x+GLKVOpXB7boddJKbwNCr+tMPSkfgCSNhl4= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U= github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index d9a28594..3aaf774c 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -47,8 +47,6 @@ func (e *Executer) Execute(input string) (bool, error) { dynamicValues := make(map[string]interface{}) previous := make(map[string]interface{}) for _, req := range e.requests { - req := req - err := req.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { ID := req.GetID() if ID != "" { diff --git a/v2/pkg/protocols/common/expressions/variables.go b/v2/pkg/protocols/common/expressions/variables.go index 48ac546e..26b56631 100644 --- a/v2/pkg/protocols/common/expressions/variables.go +++ b/v2/pkg/protocols/common/expressions/variables.go @@ -10,47 +10,55 @@ var unresolvedVariablesRegex = regexp.MustCompile(`(?:%7[B|b]|\{){2}([^}]+)(?:%7 // ContainsUnresolvedVariables returns an error with variable names if the passed // input contains unresolved {{}} variables. -func ContainsUnresolvedVariables(data string) error { - matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) - if len(matches) == 0 { - return nil - } - errorString := &strings.Builder{} - errorString.WriteString("unresolved variables found: ") - - for i, match := range matches { - if len(match) < 2 { - continue +func ContainsUnresolvedVariables(items ...string) error { + for _, data := range items { + matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) + if len(matches) == 0 { + return nil } - errorString.WriteString(match[1]) - if i != len(matches)-1 { - errorString.WriteString(",") - } - } - errorMessage := errorString.String() - return errors.New(errorMessage) -} + errorString := &strings.Builder{} + errorString.WriteString("unresolved variables found: ") -func ContainsVariablesWithNames(data string, names map[string]interface{}) error { - matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) - if len(matches) == 0 { - return nil - } - errorString := &strings.Builder{} - errorString.WriteString("unresolved variables with values found: ") - - for i, match := range matches { - if len(match) < 2 { - continue - } - matchName := match[1] - if _, ok := names[matchName]; !ok { - errorString.WriteString(matchName) + for i, match := range matches { + if len(match) < 2 { + continue + } + errorString.WriteString(match[1]) if i != len(matches)-1 { errorString.WriteString(",") } } + errorMessage := errorString.String() + return errors.New(errorMessage) } - errorMessage := errorString.String() - return errors.New(errorMessage) + + return nil +} + +func ContainsVariablesWithNames(names map[string]interface{}, items ...string) error { + for _, data := range items { + matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) + if len(matches) == 0 { + return nil + } + errorString := &strings.Builder{} + errorString.WriteString("unresolved variables with values found: ") + + for i, match := range matches { + if len(match) < 2 { + continue + } + matchName := match[1] + if _, ok := names[matchName]; !ok { + errorString.WriteString(matchName) + if i != len(matches)-1 { + errorString.WriteString(",") + } + } + } + errorMessage := errorString.String() + return errors.New(errorMessage) + } + + return nil } diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index a630762f..7cde7fed 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -11,6 +11,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/retryabledns" @@ -62,6 +63,15 @@ type Request struct { // - name: Use a retry of 3 to 5 generally // value: 5 Retries int `yaml:"retries,omitempty" jsonschema:"title=retries for dns request,description=Retries is the number of retries for the DNS request"` + // description: | + // Trace performs a trace operation for the target. + Trace bool `yaml:"trace,omitempty" jsonschema:"title=trace operation,description=Trace performs a trace operation for the target."` + // description: | + // TraceMaxRecursion is the number of max recursion allowed for trace operations + // examples: + // - name: Use a retry of 100 to 150 generally + // value: 100 + TraceMaxRecursion int `yaml:"trace-max-recursion,omitempty" jsonschema:"title=trace-max-recursion level for dns request,description=TraceMaxRecursion is the number of max recursion allowed for trace operations"` CompiledOperators *operators.Operators `yaml:"-"` dnsClient *retryabledns.Client @@ -96,7 +106,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { dnsClientOptions.Resolvers = request.Resolvers } // Create a dns client for the class - client, err := dnsclientpool.Get(options.Options, dnsClientOptions) + client, err := request.getDnsClient(options, nil) if err != nil { return errors.Wrap(err, "could not get dns client") } @@ -115,6 +125,28 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { return nil } +func (request *Request) getDnsClient(options *protocols.ExecuterOptions, metadata map[string]interface{}) (*retryabledns.Client, error) { + dnsClientOptions := &dnsclientpool.Configuration{ + Retries: request.Retries, + } + if len(request.Resolvers) > 0 { + if len(request.Resolvers) > 0 { + for _, resolver := range request.Resolvers { + if expressions.ContainsUnresolvedVariables(resolver) != nil { + var err error + resolver, err = expressions.Evaluate(resolver, metadata) + if err != nil { + return nil, errors.Wrap(err, "could not resolve resolvers expressions") + } + dnsClientOptions.Resolvers = append(dnsClientOptions.Resolvers, resolver) + } + } + } + dnsClientOptions.Resolvers = request.Resolvers + } + return dnsclientpool.Get(options.Options, dnsClientOptions) +} + // Requests returns the total number of requests the YAML rule will perform func (request *Request) Requests() int { return 1 diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index d6be2f7d..12de136a 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -2,6 +2,8 @@ package dns import ( "bytes" + "fmt" + "strings" "time" "github.com/miekg/dns" @@ -12,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/retryabledns" ) // Match matches a generic data response against a given matcher @@ -73,7 +76,7 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (in } // responseToDSLMap converts a DNS response to a map for use in DSL matching -func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent { +func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string, tracedata *retryabledns.TraceData) output.InternalEvent { return output.InternalEvent{ "host": host, "matched": matched, @@ -87,6 +90,7 @@ func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched strin "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, + "trace": traceToString(tracedata, false), } } @@ -126,3 +130,16 @@ func questionToString(resourceRecords []dns.Question) string { } return buffer.String() } + +func traceToString(tracedata *retryabledns.TraceData, withSteps bool) string { + buffer := &bytes.Buffer{} + if tracedata != nil { + for i, dnsRecord := range tracedata.DNSData { + if withSteps { + buffer.WriteString(fmt.Sprintf("request %d to resolver %s:\n", i, strings.Join(dnsRecord.Resolver, ","))) + } + buffer.WriteString(dnsRecord.Raw) + } + } + return buffer.String() +} diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index bb87b52e..1937660c 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -44,8 +44,8 @@ func TestResponseToDSLMap(t *testing.T) { resp.Rcode = dns.RcodeSuccess resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}}) - event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one") - require.Len(t, event, 12, "could not get correct number of items in dsl map") + event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", nil) + require.Len(t, event, 13, "could not get correct number of items in dsl map") require.Equal(t, dns.RcodeSuccess, event["rcode"], "could not get correct rcode") } @@ -76,7 +76,7 @@ func TestDNSOperatorMatch(t *testing.T) { resp.Rcode = dns.RcodeSuccess resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}}) - event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one") + event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", nil) t.Run("valid", func(t *testing.T) { matcher := &matchers.Matcher{ @@ -143,7 +143,7 @@ func TestDNSOperatorMatch(t *testing.T) { resp.Rcode = dns.RcodeSuccess resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "ONE.ONE.ONE.ONE."}}) - event := request.responseToDSLMap(req, resp, "ONE.ONE.ONE.ONE", "ONE.ONE.ONE.ONE") + event := request.responseToDSLMap(req, resp, "ONE.ONE.ONE.ONE", "ONE.ONE.ONE.ONE", nil) matcher := &matchers.Matcher{ Part: "raw", @@ -187,7 +187,7 @@ func TestDNSOperatorExtract(t *testing.T) { resp.Rcode = dns.RcodeSuccess resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}}) - event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one") + event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", nil) t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ @@ -257,7 +257,7 @@ func TestDNSMakeResult(t *testing.T) { resp.Rcode = dns.RcodeSuccess resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}}) - event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one") + event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", nil) finalEvent := &output.InternalWrappedEvent{InternalEvent: event} if request.CompiledOperators != nil { result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false) diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 97cb9ca2..614159e2 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" + "github.com/projectdiscovery/retryabledns" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) @@ -40,6 +41,14 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review return errors.Wrap(err, "could not build request") } + dnsClient := request.dnsClient + if varErr := expressions.ContainsUnresolvedVariables(request.Resolvers...); varErr != nil { + if dnsClient, varErr = request.getDnsClient(request.options, metadata); varErr != nil { + gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", request.options.TemplateID, domain, varErr) + return nil + } + } + requestString := compiledRequest.String() if varErr := expressions.ContainsUnresolvedVariables(requestString); varErr != nil { gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", request.options.TemplateID, domain, varErr) @@ -51,7 +60,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } // Send the request to the target servers - response, err := request.dnsClient.Do(compiledRequest) + response, err := dnsClient.Do(compiledRequest) if err != nil { request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) @@ -64,20 +73,33 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err) gologger.Verbose().Msgf("[%s] Sent DNS request to %s\n", request.options.TemplateID, domain) - outputEvent := request.responseToDSLMap(compiledRequest, response, input, input) + // perform trace if necessary + var tracedata *retryabledns.TraceData + if request.Trace { + tracedata, err = request.dnsClient.Trace(domain, request.question, request.TraceMaxRecursion) + if err != nil { + request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) + } + } + + outputEvent := request.responseToDSLMap(compiledRequest, response, input, input, tracedata) for k, v := range previous { outputEvent[k] = v } event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) + // TODO: dynamic values are not supported yet dumpResponse(event, request.options, response.String(), domain) + if request.Trace { + dumpTraceData(event, request.options, traceToString(tracedata, true), domain) + } callback(event) return nil } -func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response string, domain string) { +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response, domain string) { cliOptions := requestOptions.Options if cliOptions.Debug || cliOptions.DebugResponse { hexDump := false @@ -90,6 +112,19 @@ func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols. } } +func dumpTraceData(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, tracedata, domain string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { + hexDump := false + if responsehighlighter.HasBinaryContent(tracedata) { + hexDump = true + tracedata = hex.Dump([]byte(tracedata)) + } + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, tracedata, cliOptions.NoColor, hexDump) + gologger.Debug().Msgf("[%s] Dumped DNS Trace data for %s\n\n%s", requestOptions.TemplateID, domain, highlightedResponse) + } +} + // isURL tests a string to determine if it is a well-structured url or not. func isURL(toTest string) bool { if _, err := url.ParseRequestURI(toTest); err != nil { diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 3fbe3a8e..17a87588 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -252,7 +252,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } for _, input := range inputs { - if expressions.ContainsVariablesWithNames(input, map[string]interface{}{name: payload}) == nil { + if expressions.ContainsVariablesWithNames(map[string]interface{}{name: payload}, input) == nil { hasPayloadName = true break } diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 98037eed..09dcfb45 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -170,7 +170,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { if input.Type.String() != "" { continue } - if expressions.ContainsVariablesWithNames(input.Data, map[string]interface{}{name: payload}) == nil { + if expressions.ContainsVariablesWithNames(map[string]interface{}{name: payload}, input.Data) == nil { hasPayloadName = true break } From ccb588f383e200bff5bc1a3770efc393dadd59dc Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 18 Nov 2021 13:53:42 +0000 Subject: [PATCH 130/146] Auto Generate Syntax Docs + JSONSchema [Thu Nov 18 13:53:42 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 37 +++++++++++++++++++++++++++++++ nuclei-jsonschema.json | 10 +++++++++ v2/pkg/templates/templates_doc.go | 28 ++++++++++++++++------- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index ed19bd9c..7c26f350 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -2125,6 +2125,43 @@ retries: 5 ``` +
+ +
+ +
+ +trace bool + +
+
+ +Trace performs a trace operation for the target. + +
+ +
+ +
+ +trace-max-recursion int + +
+
+ +TraceMaxRecursion is the number of max recursion allowed for trace operations + + + +Examples: + + +```yaml +# Use a retry of 100 to 150 generally +trace-max-recursion: 100 +``` + +
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 9ace729a..47714a35 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -395,6 +395,16 @@ "title": "retries for dns request", "description": "Retries is the number of retries for the DNS request" }, + "trace": { + "type": "boolean", + "title": "trace operation", + "description": "Trace performs a trace operation for the target." + }, + "trace-max-recursion": { + "type": "integer", + "title": "trace-max-recursion level for dns request", + "description": "TraceMaxRecursion is the number of max recursion allowed for trace operations" + }, "recursion": { "type": "boolean", "title": "recurse all servers", diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 52c5da75..f6410282 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -757,7 +757,7 @@ func init() { FieldName: "dns", }, } - DNSRequestDoc.Fields = make([]encoder.Doc, 10) + DNSRequestDoc.Fields = make([]encoder.Doc, 12) DNSRequestDoc.Fields[0].Name = "matchers" DNSRequestDoc.Fields[0].Type = "[]matchers.Matcher" DNSRequestDoc.Fields[0].Note = "" @@ -825,16 +825,28 @@ func init() { DNSRequestDoc.Fields[7].Comments[encoder.LineComment] = "Retries is the number of retries for the DNS request" DNSRequestDoc.Fields[7].AddExample("Use a retry of 3 to 5 generally", 5) - DNSRequestDoc.Fields[8].Name = "recursion" + DNSRequestDoc.Fields[8].Name = "trace" DNSRequestDoc.Fields[8].Type = "bool" DNSRequestDoc.Fields[8].Note = "" - DNSRequestDoc.Fields[8].Description = "Recursion determines if resolver should recurse all records to get fresh results." - DNSRequestDoc.Fields[8].Comments[encoder.LineComment] = "Recursion determines if resolver should recurse all records to get fresh results." - DNSRequestDoc.Fields[9].Name = "resolvers" - DNSRequestDoc.Fields[9].Type = "[]string" + DNSRequestDoc.Fields[8].Description = "Trace performs a trace operation for the target." + DNSRequestDoc.Fields[8].Comments[encoder.LineComment] = "Trace performs a trace operation for the target." + DNSRequestDoc.Fields[9].Name = "trace-max-recursion" + DNSRequestDoc.Fields[9].Type = "int" DNSRequestDoc.Fields[9].Note = "" - DNSRequestDoc.Fields[9].Description = "Resolvers to use for the dns requests" - DNSRequestDoc.Fields[9].Comments[encoder.LineComment] = " Resolvers to use for the dns requests" + DNSRequestDoc.Fields[9].Description = "TraceMaxRecursion is the number of max recursion allowed for trace operations" + DNSRequestDoc.Fields[9].Comments[encoder.LineComment] = "TraceMaxRecursion is the number of max recursion allowed for trace operations" + + DNSRequestDoc.Fields[9].AddExample("Use a retry of 100 to 150 generally", 100) + DNSRequestDoc.Fields[10].Name = "recursion" + DNSRequestDoc.Fields[10].Type = "bool" + DNSRequestDoc.Fields[10].Note = "" + DNSRequestDoc.Fields[10].Description = "Recursion determines if resolver should recurse all records to get fresh results." + DNSRequestDoc.Fields[10].Comments[encoder.LineComment] = "Recursion determines if resolver should recurse all records to get fresh results." + DNSRequestDoc.Fields[11].Name = "resolvers" + DNSRequestDoc.Fields[11].Type = "[]string" + DNSRequestDoc.Fields[11].Note = "" + DNSRequestDoc.Fields[11].Description = "Resolvers to use for the dns requests" + DNSRequestDoc.Fields[11].Comments[encoder.LineComment] = " Resolvers to use for the dns requests" FILERequestDoc.Type = "file.Request" FILERequestDoc.Comments[encoder.LineComment] = " Request contains a File matching mechanism for local disk operations." From 2856e7e247d9a2ff3a6b067091847963f51f8eaa Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 18 Nov 2021 14:11:10 -0600 Subject: [PATCH 131/146] Extractor + Headless Actions structures to enums (#1217) * enum support for extractor --- v2/pkg/operators/extractors/compile.go | 10 +- .../operators/extractors/extractor_types.go | 105 ++++++++++ v2/pkg/operators/extractors/extractors.go | 29 +-- v2/pkg/protocols/dns/operators_test.go | 6 +- v2/pkg/protocols/dns/request_test.go | 2 +- v2/pkg/protocols/file/operators_test.go | 6 +- v2/pkg/protocols/file/request_test.go | 2 +- v2/pkg/protocols/headless/engine/action.go | 108 +--------- .../protocols/headless/engine/action_types.go | 185 ++++++++++++++++++ .../protocols/headless/engine/page_actions.go | 4 +- .../headless/engine/page_actions_test.go | 119 +++++------ v2/pkg/protocols/http/operators_test.go | 14 +- v2/pkg/protocols/network/operators_test.go | 6 +- v2/pkg/protocols/network/request_test.go | 2 +- .../protocols/offlinehttp/operators_test.go | 6 +- v2/pkg/templates/templates_doc_examples.go | 4 +- 16 files changed, 382 insertions(+), 226 deletions(-) create mode 100644 v2/pkg/operators/extractors/extractor_types.go create mode 100644 v2/pkg/protocols/headless/engine/action_types.go diff --git a/v2/pkg/operators/extractors/compile.go b/v2/pkg/operators/extractors/compile.go index 56771667..1d03184b 100644 --- a/v2/pkg/operators/extractors/compile.go +++ b/v2/pkg/operators/extractors/compile.go @@ -10,13 +10,12 @@ import ( // CompileExtractors performs the initial setup operation on an extractor func (e *Extractor) CompileExtractors() error { - var ok bool // Set up the extractor type - e.extractorType, ok = ExtractorTypes[e.Type] - if !ok { + computedType, err := toExtractorTypes(e.GetType().String()) + if err != nil { return fmt.Errorf("unknown extractor type specified: %s", e.Type) } - + e.extractorType = computedType // Compile the regexes for _, regex := range e.Regex { compiled, err := regexp.Compile(regex) @@ -25,7 +24,6 @@ func (e *Extractor) CompileExtractors() error { } e.regexCompiled = append(e.regexCompiled, compiled) } - for i, kval := range e.KVal { e.KVal[i] = strings.ToLower(kval) } @@ -43,7 +41,7 @@ func (e *Extractor) CompileExtractors() error { } if e.CaseInsensitive { - if e.Type != "kval" { + if e.GetType() != KValExtractor { return fmt.Errorf("case-insensitive flag is supported only for 'kval' extractors (not '%s')", e.Type) } for i := range e.KVal { diff --git a/v2/pkg/operators/extractors/extractor_types.go b/v2/pkg/operators/extractors/extractor_types.go new file mode 100644 index 00000000..227001a1 --- /dev/null +++ b/v2/pkg/operators/extractors/extractor_types.go @@ -0,0 +1,105 @@ +package extractors + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/alecthomas/jsonschema" +) + +// ExtractorType is the type of the extractor specified +type ExtractorType int + +const ( + // RegexExtractor extracts responses with regexes + RegexExtractor ExtractorType = iota + 1 + // KValExtractor extracts responses with key:value + KValExtractor + // XPathExtractor extracts responses with Xpath selectors + XPathExtractor + // JSONExtractor extracts responses with json + JSONExtractor + //limit + limit +) + +// extractorMappings is a table for conversion of extractor type from string. +var extractorMappings = map[ExtractorType]string{ + RegexExtractor: "regex", + KValExtractor: "kval", + XPathExtractor: "xpath", + JSONExtractor: "json", +} + +// GetType returns the type of the matcher +func (e *Extractor) GetType() ExtractorType { + return e.Type.ExtractorType +} + +// GetSupportedExtractorTypes returns list of supported types +func GetSupportedExtractorTypes() []ExtractorType { + var result []ExtractorType + for index := ExtractorType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toExtractorTypes(valueToMap string) (ExtractorType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range extractorMappings { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("Invalid extractor type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToLower(value)) +} + +func (t ExtractorType) String() string { + return extractorMappings[t] +} + +// TypeHolder is used to hold internal type of the extractor +type TypeHolder struct { + ExtractorType ExtractorType +} + +func (holder TypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "type of the extractor", + Description: "Type of the extractor", + } + for _, types := range GetSupportedExtractorTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *TypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toExtractorTypes(marshalledTypes) + if err != nil { + return err + } + + holder.ExtractorType = computedType + return nil +} + +func (holder *TypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.ExtractorType.String()) +} + +func (holder TypeHolder) MarshalYAML() (interface{}, error) { + return holder.ExtractorType.String(), nil +} diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index d4b2bb3c..a2121efa 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -21,7 +21,7 @@ type Extractor struct { // - "kval" // - "json" // - "xpath" - Type string `yaml:"type" jsonschema:"title=type of the extractor,description=Type of the extractor,enum=regex,enum=kval,enum=json,enum=xpath"` + Type TypeHolder `json:"name,omitempty" yaml:"type"` // extractorType is the internal type of the extractor extractorType ExtractorType @@ -113,30 +113,3 @@ type Extractor struct { // - true CaseInsensitive bool `yaml:"case-insensitive,omitempty" jsonschema:"title=use case insensitive extract,description=use case insensitive extract"` } - -// ExtractorType is the type of the extractor specified -type ExtractorType = int - -const ( - // RegexExtractor extracts responses with regexes - RegexExtractor ExtractorType = iota + 1 - // KValExtractor extracts responses with key:value - KValExtractor - // XPathExtractor extracts responses with Xpath selectors - XPathExtractor - // JSONExtractor extracts responses with json - JSONExtractor -) - -// ExtractorTypes is a table for conversion of extractor type from string. -var ExtractorTypes = map[string]ExtractorType{ - "regex": RegexExtractor, - "kval": KValExtractor, - "xpath": XPathExtractor, - "json": JSONExtractor, -} - -// GetType returns the type of the matcher -func (e *Extractor) GetType() ExtractorType { - return e.extractorType -} diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index 1937660c..a34d6a62 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -192,7 +192,7 @@ func TestDNSOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ Part: "raw", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } err = extractor.CompileExtractors() @@ -205,7 +205,7 @@ func TestDNSOperatorExtract(t *testing.T) { t.Run("kval", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "kval", + Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"rcode"}, } err = extractor.CompileExtractors() @@ -238,7 +238,7 @@ func TestDNSMakeResult(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "raw", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/dns/request_test.go b/v2/pkg/protocols/dns/request_test.go index 38827f81..138fee6f 100644 --- a/v2/pkg/protocols/dns/request_test.go +++ b/v2/pkg/protocols/dns/request_test.go @@ -35,7 +35,7 @@ func TestDNSExecuteWithResults(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "raw", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index f85ff6c8..1f0b6da6 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -154,7 +154,7 @@ func TestFileOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ Part: "raw", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } err = extractor.CompileExtractors() @@ -167,7 +167,7 @@ func TestFileOperatorExtract(t *testing.T) { t.Run("kval", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "kval", + Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"raw"}, } err = extractor.CompileExtractors() @@ -250,7 +250,7 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi Matchers: matchers, Extractors: []*extractors.Extractor{{ Part: "raw", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/file/request_test.go b/v2/pkg/protocols/file/request_test.go index bccfae72..fa0c8120 100644 --- a/v2/pkg/protocols/file/request_test.go +++ b/v2/pkg/protocols/file/request_test.go @@ -37,7 +37,7 @@ func TestFileExecuteWithResults(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "raw", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/headless/engine/action.go b/v2/pkg/protocols/headless/engine/action.go index 5f97217b..861f67a5 100644 --- a/v2/pkg/protocols/headless/engine/action.go +++ b/v2/pkg/protocols/headless/engine/action.go @@ -2,110 +2,6 @@ package engine import "strings" -// ActionType defines the action type for a browser action -type ActionType int8 - -// Types to be executed by the user. -const ( - // ActionNavigate performs a navigation to the specified URL - // URL can include nuclei payload data such as URL, Hostname, etc. - ActionNavigate ActionType = iota + 1 - // ActionScript executes a JS snippet on the page. - ActionScript - // ActionClick performs the left-click action on an Element. - ActionClick - // ActionRightClick performs the right-click action on an Element. - ActionRightClick - // ActionTextInput performs an action for a text input - ActionTextInput - // ActionScreenshot performs the screenshot action writing to a file. - ActionScreenshot - // ActionTimeInput performs an action on a time input. - ActionTimeInput - // ActionSelectInput performs an action on a select input. - ActionSelectInput - // ActionFilesInput performs an action on a file input. - ActionFilesInput - // ActionWaitLoad waits for the page to stop loading. - ActionWaitLoad - // ActionGetResource performs a get resource action on an element - ActionGetResource - // ActionExtract performs an extraction on an element - ActionExtract - // ActionSetMethod sets the request method - ActionSetMethod - // ActionAddHeader adds a header to the request - ActionAddHeader - // ActionSetHeader sets a header in the request - ActionSetHeader - // ActionDeleteHeader deletes a header from the request - ActionDeleteHeader - // ActionSetBody sets the value of the request body - ActionSetBody - // ActionWaitEvent waits for a specific event. - ActionWaitEvent - // ActionKeyboard performs a keyboard action event on a page. - ActionKeyboard - // ActionDebug debug slows down headless and adds a sleep to each page. - ActionDebug - // ActionSleep executes a sleep for a specified duration - ActionSleep - // ActionWaitVisible waits until an element appears. - ActionWaitVisible -) - -// ActionStringToAction converts an action from string to internal representation -var ActionStringToAction = map[string]ActionType{ - "navigate": ActionNavigate, - "script": ActionScript, - "click": ActionClick, - "rightclick": ActionRightClick, - "text": ActionTextInput, - "screenshot": ActionScreenshot, - "time": ActionTimeInput, - "select": ActionSelectInput, - "files": ActionFilesInput, - "waitload": ActionWaitLoad, - "getresource": ActionGetResource, - "extract": ActionExtract, - "setmethod": ActionSetMethod, - "addheader": ActionAddHeader, - "setheader": ActionSetHeader, - "deleteheader": ActionDeleteHeader, - "setbody": ActionSetBody, - "waitevent": ActionWaitEvent, - "keyboard": ActionKeyboard, - "debug": ActionDebug, - "sleep": ActionSleep, - "waitvisible": ActionWaitVisible, -} - -// ActionToActionString converts an action from internal representation to string -var ActionToActionString = map[ActionType]string{ - ActionNavigate: "navigate", - ActionScript: "script", - ActionClick: "click", - ActionRightClick: "rightclick", - ActionTextInput: "text", - ActionScreenshot: "screenshot", - ActionTimeInput: "time", - ActionSelectInput: "select", - ActionFilesInput: "files", - ActionWaitLoad: "waitload", - ActionGetResource: "getresource", - ActionExtract: "extract", - ActionSetMethod: "set-method", - ActionAddHeader: "addheader", - ActionSetHeader: "setheader", - ActionDeleteHeader: "deleteheader", - ActionSetBody: "setbody", - ActionWaitEvent: "waitevent", - ActionKeyboard: "keyboard", - ActionDebug: "debug", - ActionSleep: "sleep", - ActionWaitVisible: "waitvisible", -} - // Action is an action taken by the browser to reach a navigation // // Each step that the browser executes is an action. Most navigations @@ -152,13 +48,13 @@ type Action struct { // - "keyboard" // - "debug" // - "sleep" - ActionType string `yaml:"action" jsonschema:"title=action to perform,description=Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"` + ActionType ActionTypeHolder `yaml:"action" jsonschema:"title=action to perform,description=Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"` } // String returns the string representation of an action func (a *Action) String() string { builder := &strings.Builder{} - builder.WriteString(a.ActionType) + builder.WriteString(a.ActionType.String()) if a.Name != "" { builder.WriteString(" Name:") builder.WriteString(a.Name) diff --git a/v2/pkg/protocols/headless/engine/action_types.go b/v2/pkg/protocols/headless/engine/action_types.go new file mode 100644 index 00000000..4f71b421 --- /dev/null +++ b/v2/pkg/protocols/headless/engine/action_types.go @@ -0,0 +1,185 @@ +package engine + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/alecthomas/jsonschema" +) + +// ActionType defines the action type for a browser action +type ActionType int8 + +// Types to be executed by the user. +const ( + // ActionNavigate performs a navigation to the specified URL + // URL can include nuclei payload data such as URL, Hostname, etc. + ActionNavigate ActionType = iota + 1 + // ActionScript executes a JS snippet on the page. + ActionScript + // ActionClick performs the left-click action on an Element. + ActionClick + // ActionRightClick performs the right-click action on an Element. + ActionRightClick + // ActionTextInput performs an action for a text input + ActionTextInput + // ActionScreenshot performs the screenshot action writing to a file. + ActionScreenshot + // ActionTimeInput performs an action on a time input. + ActionTimeInput + // ActionSelectInput performs an action on a select input. + ActionSelectInput + // ActionFilesInput performs an action on a file input. + ActionFilesInput + // ActionWaitLoad waits for the page to stop loading. + ActionWaitLoad + // ActionGetResource performs a get resource action on an element + ActionGetResource + // ActionExtract performs an extraction on an element + ActionExtract + // ActionSetMethod sets the request method + ActionSetMethod + // ActionAddHeader adds a header to the request + ActionAddHeader + // ActionSetHeader sets a header in the request + ActionSetHeader + // ActionDeleteHeader deletes a header from the request + ActionDeleteHeader + // ActionSetBody sets the value of the request body + ActionSetBody + // ActionWaitEvent waits for a specific event. + ActionWaitEvent + // ActionKeyboard performs a keyboard action event on a page. + ActionKeyboard + // ActionDebug debug slows down headless and adds a sleep to each page. + ActionDebug + // ActionSleep executes a sleep for a specified duration + ActionSleep + // ActionWaitVisible waits until an element appears. + ActionWaitVisible + // limit + limit +) + +// ActionStringToAction converts an action from string to internal representation +var ActionStringToAction = map[string]ActionType{ + "navigate": ActionNavigate, + "script": ActionScript, + "click": ActionClick, + "rightclick": ActionRightClick, + "text": ActionTextInput, + "screenshot": ActionScreenshot, + "time": ActionTimeInput, + "select": ActionSelectInput, + "files": ActionFilesInput, + "waitload": ActionWaitLoad, + "getresource": ActionGetResource, + "extract": ActionExtract, + "setmethod": ActionSetMethod, + "addheader": ActionAddHeader, + "setheader": ActionSetHeader, + "deleteheader": ActionDeleteHeader, + "setbody": ActionSetBody, + "waitevent": ActionWaitEvent, + "keyboard": ActionKeyboard, + "debug": ActionDebug, + "sleep": ActionSleep, + "waitvisible": ActionWaitVisible, +} + +// ActionToActionString converts an action from internal representation to string +var ActionToActionString = map[ActionType]string{ + ActionNavigate: "navigate", + ActionScript: "script", + ActionClick: "click", + ActionRightClick: "rightclick", + ActionTextInput: "text", + ActionScreenshot: "screenshot", + ActionTimeInput: "time", + ActionSelectInput: "select", + ActionFilesInput: "files", + ActionWaitLoad: "waitload", + ActionGetResource: "getresource", + ActionExtract: "extract", + ActionSetMethod: "set-method", + ActionAddHeader: "addheader", + ActionSetHeader: "setheader", + ActionDeleteHeader: "deleteheader", + ActionSetBody: "setbody", + ActionWaitEvent: "waitevent", + ActionKeyboard: "keyboard", + ActionDebug: "debug", + ActionSleep: "sleep", + ActionWaitVisible: "waitvisible", +} + +// GetSupportedActionTypes returns list of supported types +func GetSupportedActionTypes() []ActionType { + var result []ActionType + for index := ActionType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toActionTypes(valueToMap string) (ActionType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range ActionToActionString { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("Invalid action type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToLower(value)) +} + +func (t ActionType) String() string { + return ActionToActionString[t] +} + +// ActionTypeHolder is used to hold internal type of the action +type ActionTypeHolder struct { + ActionType ActionType +} + +func (holder ActionTypeHolder) String() string { + return holder.ActionType.String() +} +func (holder ActionTypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "action to perform", + Description: "Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep", + } + for _, types := range GetSupportedActionTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *ActionTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toActionTypes(marshalledTypes) + if err != nil { + return err + } + + holder.ActionType = computedType + return nil +} + +func (holder *ActionTypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.ActionType.String()) +} + +func (holder ActionTypeHolder) MarshalYAML() (interface{}, error) { + return holder.ActionType.String(), nil +} diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index fc6e821e..234d726e 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -24,9 +24,7 @@ func (p *Page) ExecuteActions(baseURL *url.URL, actions []*Action) (map[string]s outData := make(map[string]string) for _, act := range actions { - actionType := ActionStringToAction[act.ActionType] - - switch actionType { + switch act.ActionType.ActionType { case ActionNavigate: err = p.NavigateURL(act, outData, baseURL) case ActionScript: diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 902b978a..f9819756 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -28,7 +28,7 @@ func TestActionNavigate(t *testing.T) { ` - actions := []*Action{{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}} + actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}} testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { require.Nil(t, err, "could not run page actions") @@ -50,10 +50,11 @@ func TestActionScript(t *testing.T) { t.Run("run-and-results", func(t *testing.T) { actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Name: "test", Data: map[string]string{"code": "window.test"}}, } + testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { require.Nil(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") @@ -63,10 +64,10 @@ func TestActionScript(t *testing.T) { t.Run("hook", func(t *testing.T) { actions := []*Action{ - {ActionType: "script", Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Name: "test", Data: map[string]string{"code": "window.test"}}, } testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { require.Nil(t, err, "could not run page actions") @@ -87,9 +88,9 @@ func TestActionClick(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "click", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionClick)}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -120,9 +121,9 @@ func TestActionRightClick(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "rightclick", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionRightClick)}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -145,9 +146,9 @@ func TestActionTextInput(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "text", Data: map[string]string{"selector": "input", "value": "test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionTextInput)}, Data: map[string]string{"selector": "input", "value": "test"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -162,9 +163,9 @@ func TestActionTextInput(t *testing.T) { func TestActionHeadersChange(t *testing.T) { actions := []*Action{ - {ActionType: "setheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetHeader)}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -189,9 +190,9 @@ func TestActionScreenshot(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "screenshot", Data: map[string]string{"to": "test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScreenshot)}, Data: map[string]string{"to": "test"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -214,9 +215,9 @@ func TestActionTimeInput(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "time", Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionTimeInput)}, Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -242,9 +243,9 @@ func TestActionSelectInput(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "select", Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSelectInput)}, Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -265,9 +266,9 @@ func TestActionFilesInput(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "files", Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionFilesInput)}, Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -291,8 +292,8 @@ func TestActionWaitLoad(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -316,8 +317,8 @@ func TestActionGetResource(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "getresource", Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionGetResource)}, Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -336,8 +337,8 @@ func TestActionExtract(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "extract", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionExtract)}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -355,8 +356,8 @@ func TestActionSetMethod(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "setmethod", Data: map[string]string{"part": "x", "method": "SET"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetMethod)}, Data: map[string]string{"part": "x", "method": "SET"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -367,9 +368,9 @@ func TestActionSetMethod(t *testing.T) { func TestActionAddHeader(t *testing.T) { actions := []*Action{ - {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -386,11 +387,11 @@ func TestActionAddHeader(t *testing.T) { func TestActionDeleteHeader(t *testing.T) { actions := []*Action{ - {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}}, - {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}}, - {ActionType: "deleteheader", Data: map[string]string{"part": "request", "key": "Test2"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionDeleteHeader)}, Data: map[string]string{"part": "request", "key": "Test2"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -407,9 +408,9 @@ func TestActionDeleteHeader(t *testing.T) { func TestActionSetBody(t *testing.T) { actions := []*Action{ - {ActionType: "setbody", Data: map[string]string{"part": "request", "body": "hello"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetBody)}, Data: map[string]string{"part": "request", "body": "hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -435,10 +436,10 @@ func TestActionKeyboard(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "click", Data: map[string]string{"selector": "input"}}, - {ActionType: "keyboard", Data: map[string]string{"keys": "Test2"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionClick)}, Data: map[string]string{"selector": "input"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionKeyboard)}, Data: map[string]string{"keys": "Test2"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -461,8 +462,8 @@ func TestActionSleep(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "sleep", Data: map[string]string{"duration": "2"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSleep)}, Data: map[string]string{"duration": "2"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -484,8 +485,8 @@ func TestActionWaitVisible(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitVisible)}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, } t.Run("wait for an element being visible", func(t *testing.T) { diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index 7950bc64..50e9c9d0 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -166,7 +166,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ Part: "body", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } err = extractor.CompileExtractors() @@ -179,7 +179,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("kval", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "kval", + Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"test_header"}, } err = extractor.CompileExtractors() @@ -195,7 +195,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("jq-simple", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "json", + Type: extractors.TypeHolder{ExtractorType: extractors.JSONExtractor}, JSON: []string{".batters | .batter | .[] | .id"}, } err = extractor.CompileExtractors() @@ -207,7 +207,7 @@ func TestHTTPOperatorExtract(t *testing.T) { }) t.Run("jq-array", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "json", + Type: extractors.TypeHolder{ExtractorType: extractors.JSONExtractor}, JSON: []string{".array"}, } err = extractor.CompileExtractors() @@ -219,7 +219,7 @@ func TestHTTPOperatorExtract(t *testing.T) { }) t.Run("jq-object", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "json", + Type: extractors.TypeHolder{ExtractorType: extractors.JSONExtractor}, JSON: []string{".batters"}, } err = extractor.CompileExtractors() @@ -235,7 +235,7 @@ func TestHTTPOperatorExtract(t *testing.T) { event["body"] = exampleResponseBody extractor := &extractors.Extractor{ - Type: "kval", + Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"TEST_HEADER"}, // only applies to KVal CaseInsensitive: true, } @@ -267,7 +267,7 @@ func TestHTTPMakeResult(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "body", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index 9e8a193c..292473bf 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -149,7 +149,7 @@ func TestNetworkOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ Part: "data", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } err = extractor.CompileExtractors() @@ -162,7 +162,7 @@ func TestNetworkOperatorExtract(t *testing.T) { t.Run("kval", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "kval", + Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"request"}, } err = extractor.CompileExtractors() @@ -193,7 +193,7 @@ func TestNetworkMakeResult(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "data", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index 0e8d194b..1e51b9c6 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -38,7 +38,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "data", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"

.*

"}, }}, }, diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index 05fe9468..1290dfb7 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -139,7 +139,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ Part: "body", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } err = extractor.CompileExtractors() @@ -152,7 +152,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("kval", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "kval", + Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"test-header"}, Part: "header", } @@ -184,7 +184,7 @@ func TestHTTPMakeResult(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "body", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }} diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index 88d5e5e8..c2976657 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -44,7 +44,7 @@ var ( Recursion: true, Operators: operators.Operators{ Extractors: []*extractors.Extractor{ - {Type: "regex", Regex: []string{"ec2-[-\\d]+\\.compute[-\\d]*\\.amazonaws\\.com", "ec2-[-\\d]+\\.[\\w\\d\\-]+\\.compute[-\\d]*\\.amazonaws\\.com"}}, + {Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"ec2-[-\\d]+\\.compute[-\\d]*\\.amazonaws\\.com", "ec2-[-\\d]+\\.[\\w\\d\\-]+\\.compute[-\\d]*\\.amazonaws\\.com"}}, }, }, } @@ -54,7 +54,7 @@ var ( Extensions: []string{"all"}, Operators: operators.Operators{ Extractors: []*extractors.Extractor{ - {Type: "regex", Regex: []string{"amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"}}, + {Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"}}, }, }, } From 5a0f483cd3d59f86fef956b815ba4dfffaa4b676 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 18 Nov 2021 20:12:46 +0000 Subject: [PATCH 132/146] Auto Generate Syntax Docs + JSONSchema [Thu Nov 18 20:12:46 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 4 +- nuclei-jsonschema.json | 77 ++++++++++++++++++------------- v2/pkg/templates/templates_doc.go | 4 +- 3 files changed, 48 insertions(+), 37 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 7c26f350..6ceae433 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -1658,7 +1658,7 @@ name: cookie-extractor
-type string +type TypeHolder
@@ -2879,7 +2879,7 @@ Description is the optional description of the headless action
-action string +action ActionTypeHolder
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 47714a35..ddaf889f 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -130,15 +130,8 @@ "description": "Name of the extractor" }, "type": { - "enum": [ - "regex", - "kval", - "json", - "xpath" - ], - "type": "string", - "title": "type of the extractor", - "description": "Type of the extractor" + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/extractors.TypeHolder" }, "regex": { "items": { @@ -201,6 +194,17 @@ "additionalProperties": false, "type": "object" }, + "extractors.TypeHolder": { + "enum": [ + "regex", + "kval", + "xpath", + "json" + ], + "type": "string", + "title": "type of the extractor", + "description": "Type of the extractor" + }, "matchers.Matcher": { "required": [ "type" @@ -555,30 +559,8 @@ "description": "Description of the headless action" }, "action": { - "enum": [ - "navigate", - "script", - "click", - "rightclick", - "text", - "screenshot", - "time", - "select", - "files", - "waitload", - "getresource", - "extract", - "setmethod", - "addheader", - "setheader", - "deleteheader", - "setbody", - "waitevent", - "keyboard", - "debug", - "sleep" - ], - "type": "string", + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/engine.ActionTypeHolder", "title": "action to perform", "description": "Type of actions to perform" } @@ -586,6 +568,35 @@ "additionalProperties": false, "type": "object" }, + "engine.ActionTypeHolder": { + "enum": [ + "navigate", + "script", + "click", + "rightclick", + "text", + "screenshot", + "time", + "select", + "files", + "waitload", + "getresource", + "extract", + "set-method", + "addheader", + "setheader", + "deleteheader", + "setbody", + "waitevent", + "keyboard", + "debug", + "sleep", + "waitvisible" + ], + "type": "string", + "title": "action to perform", + "description": "Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep" + }, "http.HTTPMethodTypeHolder": { "enum": [ "GET", diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index f6410282..df4ff0a4 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -649,7 +649,7 @@ func init() { EXTRACTORSExtractorDoc.Fields[0].AddExample("", "cookie-extractor") EXTRACTORSExtractorDoc.Fields[1].Name = "type" - EXTRACTORSExtractorDoc.Fields[1].Type = "string" + EXTRACTORSExtractorDoc.Fields[1].Type = "TypeHolder" EXTRACTORSExtractorDoc.Fields[1].Note = "" EXTRACTORSExtractorDoc.Fields[1].Description = "Type is the type of the extractor." EXTRACTORSExtractorDoc.Fields[1].Comments[encoder.LineComment] = "Type is the type of the extractor." @@ -1098,7 +1098,7 @@ func init() { ENGINEActionDoc.Fields[2].Description = "Description is the optional description of the headless action" ENGINEActionDoc.Fields[2].Comments[encoder.LineComment] = "Description is the optional description of the headless action" ENGINEActionDoc.Fields[3].Name = "action" - ENGINEActionDoc.Fields[3].Type = "string" + ENGINEActionDoc.Fields[3].Type = "ActionTypeHolder" ENGINEActionDoc.Fields[3].Note = "" ENGINEActionDoc.Fields[3].Description = "Action is the type of the action to perform." ENGINEActionDoc.Fields[3].Comments[encoder.LineComment] = "Action is the type of the action to perform." From 883cb22c39f0eb70a8aba2ad27f59c48912a6bcf Mon Sep 17 00:00:00 2001 From: sandeep Date: Fri, 19 Nov 2021 14:46:10 +0530 Subject: [PATCH 133/146] Adding more resources --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e4626a05..efe80f38 100644 --- a/README.md +++ b/README.md @@ -280,6 +280,8 @@ We have [a discussion thread around this](https://github.com/projectdiscovery/nu ### Resources + +- [Scanning Live Web Applications with Nuclei in CI/CD Pipeline](https://blog.escape.tech/devsecops-part-iii-scanning-live-web-applications/) by [@TristanKalos](https://twitter.com/TristanKalos) - [Community Powered Scanning with Nuclei](https://blog.projectdiscovery.io/community-powered-scanning-with-nuclei/) - [Nuclei Unleashed - Quickly write complex exploits](https://blog.projectdiscovery.io/nuclei-unleashed-quickly-write-complex-exploits/) - [Nuclei - Fuzz all the things](https://blog.projectdiscovery.io/nuclei-fuzz-all-the-things/) From 09eba6c0cf7b27c8210eb327f0b2c7a6e5e8142c Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 19 Nov 2021 12:15:43 +0200 Subject: [PATCH 134/146] feat: In case of binary data, show a hexadecimal view as well #1080 (#1266) Highlight the longest occurrences when there are matches that are substrings of each other --- .../response_highlighter.go | 40 +++++++--- .../response_highlighter_test.go | 77 +++++++++++++------ 2 files changed, 84 insertions(+), 33 deletions(-) diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go index f637644f..ccec3cd3 100644 --- a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go @@ -1,6 +1,7 @@ package responsehighlighter import ( + "sort" "strconv" "strings" @@ -14,18 +15,14 @@ var colorFunction = aurora.Green func Highlight(operatorResult *operators.Result, response string, noColor, hexDump bool) string { result := response if operatorResult != nil && !noColor { - for _, matches := range operatorResult.Matches { - if len(matches) > 0 { - for _, currentMatch := range matches { - if hexDump { - highlightedHexDump, err := toHighLightedHexDump(result, currentMatch) - if err == nil { - result = highlightedHexDump.String() - } - } else { - result = strings.ReplaceAll(result, currentMatch, addColor(currentMatch)) - } + for _, currentMatch := range getSortedMatches(operatorResult) { + if hexDump { + highlightedHexDump, err := toHighLightedHexDump(result, currentMatch) + if err == nil { + result = highlightedHexDump.String() } + } else { + result = highlightASCII(currentMatch, result) } } } @@ -33,6 +30,27 @@ func Highlight(operatorResult *operators.Result, response string, noColor, hexDu return result } +func highlightASCII(currentMatch string, result string) string { + var coloredMatchBuilder strings.Builder + for _, char := range currentMatch { + coloredMatchBuilder.WriteString(addColor(string(char))) + } + + return strings.ReplaceAll(result, currentMatch, coloredMatchBuilder.String()) +} + +func getSortedMatches(operatorResult *operators.Result) []string { + sortedMatches := make([]string, 0, len(operatorResult.Matches)) + for _, matches := range operatorResult.Matches { + sortedMatches = append(sortedMatches, matches...) + } + + sort.Slice(sortedMatches, func(i, j int) bool { + return len(sortedMatches[i]) > len(sortedMatches[j]) + }) + return sortedMatches +} + func CreateStatusCodeSnippet(response string, statusCode int) string { if strings.HasPrefix(response, "HTTP/") { strStatusCode := strconv.Itoa(statusCode) diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go index a60341ef..fcdd16a8 100644 --- a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go @@ -12,15 +12,16 @@ import ( const input = "abcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmn" func TestHexDumpHighlighting(t *testing.T) { - const highlightedHexDumpResponse = `00000000 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 |abcdefghijklmnab| -00000010 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 |cdefghijklmnabcd| -00000020 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 |efghijklmnabcdef| -00000030 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 |ghijklmnabcdefgh| -00000040 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a |ijklmnabcdefghij| -00000050 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c |klmnabcdefghijkl| -00000060 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |mnabcdefghijklmn| -00000070 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |abcdefghijklmn| -` + highlightedHexDumpResponse := + "00000000 61 62 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e 61 62 |abc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmnab|\n" + + "00000010 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e 61 62 63 \x1b[32m64\x1b[0m |c\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmnabc\x1b[32md\x1b[0m|\n" + + "00000020 \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e 61 62 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m |\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmnabc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m|\n" + + "00000030 \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e 61 62 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m |\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmnabc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m|\n" + + "00000040 \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e 61 62 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m |\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmnabc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0m|\n" + + "00000050 6b 6c 6d 6e 61 62 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c |klmnabc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mkl|\n" + + "00000060 6d 6e 61 62 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e |mnabc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn|\n" + + "00000070 61 62 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e |abc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn|\n" + t.Run("Test highlighting when the snippet is wrapped", func(t *testing.T) { result, err := toHighLightedHexDump(hex.Dump([]byte(input)), "defghij") assert.Nil(t, err) @@ -31,24 +32,25 @@ func TestHexDumpHighlighting(t *testing.T) { value := "asdfasdfasda|basdfadsdfs|" result, err := toHighLightedHexDump(hex.Dump([]byte(value)), "a|b") - expected := `00000000 61 73 64 66 61 73 64 66 61 73 64 61 7c 62 61 73 |asdfasdfasda|bas| -00000010 64 66 61 64 73 64 66 73 7c |dfadsdfs|| -` + expected := + "00000000 61 73 64 66 61 73 64 66 61 73 64 \x1b[32m61\x1b[0m \x1b[32m7c\x1b[0m \x1b[32m62\x1b[0m 61 73 |asdfasdfasd\x1b[32ma\x1b[0m\x1b[32m|\x1b[0m\x1b[32mb\x1b[0mas|\n" + + "00000010 64 66 61 64 73 64 66 73 7c |dfadsdfs||\n" + assert.Nil(t, err) assert.Equal(t, expected, result.String()) }) } func TestHighlight(t *testing.T) { - const multiSnippetHighlightHexDumpResponse = `00000000 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 |abcdefghijklmnab| -00000010 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 |cdefghijklmnabcd| -00000020 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 |efghijklmnabcdef| -00000030 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 |ghijklmnabcdefgh| -00000040 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a |ijklmnabcdefghij| -00000050 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c |klmnabcdefghijkl| -00000060 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |mnabcdefghijklmn| -00000070 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |abcdefghijklmn| -` + const multiSnippetHighlightHexDumpResponse = "00000000 \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m |\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32ma\x1b[0m\x1b[32mb\x1b[0m|\n" + + "00000010 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m |c\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m|\n" + + "00000020 \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m |\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m|\n" + + "00000030 \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m |\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m|\n" + + "00000040 \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m |\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0m|\n" + + "00000050 6b 6c 6d 6e \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c |klmn\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mkl|\n" + + "00000060 6d 6e \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e |mn\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn|\n" + + "00000070 \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e |\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn|\n" + matches := map[string][]string{ "first": {"defghij"}, "second": {"ab"}, @@ -62,7 +64,17 @@ func TestHighlight(t *testing.T) { t.Run("Test highlighting without hexdump", func(t *testing.T) { result := Highlight(&operatorResult, input, false, false) - expected := `abcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmn` + expected := + "\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn" + print(result) assert.Equal(t, expected, result) }) @@ -76,3 +88,24 @@ func TestHighlight(t *testing.T) { assert.Equal(t, hex.Dump([]byte(input)), result) }) } + +func TestMultiSubstringMatchHighlight(t *testing.T) { + const input = ` +start ValueToMatch end +start ValueToMatch-1.2.3 end +start ValueToMatch-2.1 end +` + matches := map[string][]string{ + "first": {"ValueToMatch"}, + "second": {"ValueToMatch-1.2.3"}, + "third": {"ValueToMatch-2.1"}, + } + operatorResult := operators.Result{Matches: matches} + + expected := + "\nstart \x1b[32mV\x1b[0m\x1b[32ma\x1b[0m\x1b[32ml\x1b[0m\x1b[32mu\x1b[0m\x1b[32me\x1b[0m\x1b[32mT\x1b[0m\x1b[32mo\x1b[0m\x1b[32mM\x1b[0m\x1b[32ma\x1b[0m\x1b[32mt\x1b[0m\x1b[32mc\x1b[0m\x1b[32mh\x1b[0m end\n" + + "start \x1b[32mV\x1b[0m\x1b[32ma\x1b[0m\x1b[32ml\x1b[0m\x1b[32mu\x1b[0m\x1b[32me\x1b[0m\x1b[32mT\x1b[0m\x1b[32mo\x1b[0m\x1b[32mM\x1b[0m\x1b[32ma\x1b[0m\x1b[32mt\x1b[0m\x1b[32mc\x1b[0m\x1b[32mh\x1b[0m\x1b[32m-\x1b[0m\x1b[32m1\x1b[0m\x1b[32m.\x1b[0m\x1b[32m2\x1b[0m\x1b[32m.\x1b[0m\x1b[32m3\x1b[0m end\n" + + "start \x1b[32mV\x1b[0m\x1b[32ma\x1b[0m\x1b[32ml\x1b[0m\x1b[32mu\x1b[0m\x1b[32me\x1b[0m\x1b[32mT\x1b[0m\x1b[32mo\x1b[0m\x1b[32mM\x1b[0m\x1b[32ma\x1b[0m\x1b[32mt\x1b[0m\x1b[32mc\x1b[0m\x1b[32mh\x1b[0m\x1b[32m-\x1b[0m\x1b[32m2\x1b[0m\x1b[32m.\x1b[0m\x1b[32m1\x1b[0m end \n" + result := Highlight(&operatorResult, input, false, false) + assert.Equal(t, expected, result) +} From 4b8ec29d8d788d45543570ca97bbc2e654a4779a Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Fri, 19 Nov 2021 04:54:09 -0600 Subject: [PATCH 135/146] struct to enums changes for Matchers (#1246) * struct to enums changes for Matchers --- v2/pkg/operators/matchers/compile.go | 13 +- v2/pkg/operators/matchers/match_test.go | 6 +- v2/pkg/operators/matchers/matchers.go | 33 +---- v2/pkg/operators/matchers/matchers_types.go | 115 ++++++++++++++++++ v2/pkg/operators/operators.go | 2 +- v2/pkg/protocols/dns/operators_test.go | 12 +- v2/pkg/protocols/dns/request_test.go | 2 +- v2/pkg/protocols/file/operators_test.go | 12 +- v2/pkg/protocols/file/request_test.go | 2 +- v2/pkg/protocols/http/operators_test.go | 10 +- v2/pkg/protocols/network/operators_test.go | 10 +- v2/pkg/protocols/network/request_test.go | 2 +- .../protocols/offlinehttp/operators_test.go | 8 +- v2/pkg/templates/templates_doc_examples.go | 8 +- 14 files changed, 163 insertions(+), 72 deletions(-) create mode 100644 v2/pkg/operators/matchers/matchers_types.go diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index 0efebb1a..79d062a7 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -25,11 +25,18 @@ func (m *Matcher) CompileMatchers() error { } // Set up the matcher type - m.matcherType, ok = MatcherTypes[m.Type] - if !ok { + computedType, err := toMatcherTypes(m.GetType().String()) + if err != nil { return fmt.Errorf("unknown matcher type specified: %s", m.Type) } + m.matcherType = computedType + // By default, match on body if user hasn't provided any specific items + if m.Part == "" { + m.Part = "body" + } + + // Compile the regexes for _, regex := range m.Regex { compiled, err := regexp.Compile(regex) @@ -68,7 +75,7 @@ func (m *Matcher) CompileMatchers() error { } if m.CaseInsensitive { - if m.Type != "word" { + if m.GetType() != WordsMatcher { return fmt.Errorf("case-insensitive flag is supported only for 'word' matchers (not '%s')", m.Type) } for i := range m.Words { diff --git a/v2/pkg/operators/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go index 4c2f9bc0..b52adbc6 100644 --- a/v2/pkg/operators/matchers/match_test.go +++ b/v2/pkg/operators/matchers/match_test.go @@ -19,7 +19,7 @@ func TestWordANDCondition(t *testing.T) { } func TestRegexANDCondition(t *testing.T) { - m := &Matcher{Type: "regex", Condition: "and", Regex: []string{"[a-z]{3}", "\\d{2}"}} + m := &Matcher{Type: MatcherTypeHolder{MatcherType: RegexMatcher}, Condition: "and", Regex: []string{"[a-z]{3}", "\\d{2}"}} err := m.CompileMatchers() require.Nil(t, err) @@ -49,7 +49,7 @@ func TestORCondition(t *testing.T) { } func TestRegexOrCondition(t *testing.T) { - m := &Matcher{Type: "regex", Condition: "or", Regex: []string{"[a-z]{3}", "\\d{2}"}} + m := &Matcher{Type: MatcherTypeHolder{MatcherType: RegexMatcher}, Condition: "or", Regex: []string{"[a-z]{3}", "\\d{2}"}} err := m.CompileMatchers() require.Nil(t, err) @@ -63,7 +63,7 @@ func TestRegexOrCondition(t *testing.T) { } func TestHexEncoding(t *testing.T) { - m := &Matcher{Encoding: "hex", Type: "word", Part: "body", Words: []string{"50494e47"}} + m := &Matcher{Encoding: "hex", Type: MatcherTypeHolder{MatcherType: WordsMatcher}, Part: "body", Words: []string{"50494e47"}} err := m.CompileMatchers() require.Nil(t, err, "could not compile matcher") diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index 4c0fd1fe..3f37deb0 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -17,7 +17,7 @@ type Matcher struct { // - "regex" // - "binary" // - "dsl" - Type string `yaml:"type" jsonschema:"title=type of matcher,description=Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl"` + Type MatcherTypeHolder `yaml:"type" jsonschema:"title=type of matcher,description=Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl"` // description: | // Condition is the optional condition between two matcher variables. By default, // the condition is assumed to be OR. @@ -120,33 +120,6 @@ type Matcher struct { dslCompiled []*govaluate.EvaluableExpression } -// MatcherType is the type of the matcher specified -type MatcherType = int - -const ( - // WordsMatcher matches responses with words - WordsMatcher MatcherType = iota + 1 - // RegexMatcher matches responses with regexes - RegexMatcher - // BinaryMatcher matches responses with words - BinaryMatcher - // StatusMatcher matches responses with status codes - StatusMatcher - // SizeMatcher matches responses with response size - SizeMatcher - // DSLMatcher matches based upon dsl syntax - DSLMatcher -) - -// MatcherTypes is a table for conversion of matcher type from string. -var MatcherTypes = map[string]MatcherType{ - "status": StatusMatcher, - "size": SizeMatcher, - "word": WordsMatcher, - "regex": RegexMatcher, - "binary": BinaryMatcher, - "dsl": DSLMatcher, -} // ConditionType is the type of condition for matcher type ConditionType int @@ -180,7 +153,3 @@ func (m *Matcher) ResultWithMatchedSnippet(data bool, matchedSnippet []string) ( return data, matchedSnippet } -// GetType returns the type of the matcher -func (m *Matcher) GetType() MatcherType { - return m.matcherType -} diff --git a/v2/pkg/operators/matchers/matchers_types.go b/v2/pkg/operators/matchers/matchers_types.go new file mode 100644 index 00000000..312d4f59 --- /dev/null +++ b/v2/pkg/operators/matchers/matchers_types.go @@ -0,0 +1,115 @@ +package matchers + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/alecthomas/jsonschema" +) + +// MatcherType is the type of the matcher specified +type MatcherType int + +const ( + // WordsMatcher matches responses with words + WordsMatcher MatcherType = iota + 1 + // RegexMatcher matches responses with regexes + RegexMatcher + // BinaryMatcher matches responses with words + BinaryMatcher + // StatusMatcher matches responses with status codes + StatusMatcher + // SizeMatcher matches responses with response size + SizeMatcher + // DSLMatcher matches based upon dsl syntax + DSLMatcher + //limit + limit +) + +// MatcherTypes is a table for conversion of matcher type from string. +var MatcherTypes = map[MatcherType]string{ + StatusMatcher: "status", + SizeMatcher: "size", + WordsMatcher: "word", + RegexMatcher: "regex", + BinaryMatcher: "binary", + DSLMatcher: "dsl", +} + +//GetType returns the type of the matcher +func (e *Matcher) GetType() MatcherType { + return e.Type.MatcherType +} + +// GetSupportedMatcherTypes returns list of supported types +func GetSupportedMatcherTypes() []MatcherType { + var result []MatcherType + for index := MatcherType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toMatcherTypes(valueToMap string) (MatcherType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range MatcherTypes { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("Invalid matcher type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToLower(value)) +} + +func (t MatcherType) String() string { + return MatcherTypes[t] +} + +// MatcherTypeHolder is used to hold internal type of the matcher +type MatcherTypeHolder struct { + MatcherType MatcherType +} + +func (t MatcherTypeHolder) String() string { + return t.MatcherType.String() +} + +func (holder MatcherTypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "type of the matcher", + Description: "Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl", + } + for _, types := range GetSupportedMatcherTypes() { + gotType.Enum = append(gotType.Enum, types) + } + return gotType +} + +func (holder *MatcherTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toMatcherTypes(marshalledTypes) + if err != nil { + return err + } + + holder.MatcherType = computedType + return nil +} + +func (holder *MatcherTypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.MatcherType) +} + +func (holder MatcherTypeHolder) MarshalYAML() (interface{}, error) { + return holder.MatcherType, nil +} diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index af13b4d9..0b908731 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -179,7 +179,7 @@ func getMatcherName(matcher *matchers.Matcher, matcherIndex int) string { if matcher.Name != "" { return matcher.Name } else { - return matcher.Type + "-" + strconv.Itoa(matcherIndex+1) // making the index start from 1 to be more readable + return matcher.Type.String() + "-" + strconv.Itoa(matcherIndex+1) // making the index start from 1 to be more readable } } diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index a34d6a62..aef43ccf 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -81,7 +81,7 @@ func TestDNSOperatorMatch(t *testing.T) { t.Run("valid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, } err = matcher.CompileMatchers() @@ -95,7 +95,7 @@ func TestDNSOperatorMatch(t *testing.T) { t.Run("rcode", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "rcode", - Type: "status", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.StatusMatcher}, Status: []int{dns.RcodeSuccess}, } err = matcher.CompileMatchers() @@ -109,7 +109,7 @@ func TestDNSOperatorMatch(t *testing.T) { t.Run("negative", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Negative: true, Words: []string{"random"}, } @@ -124,7 +124,7 @@ func TestDNSOperatorMatch(t *testing.T) { t.Run("invalid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"random"}, } err := matcher.CompileMatchers() @@ -147,7 +147,7 @@ func TestDNSOperatorMatch(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"one.ONE.one.ONE"}, CaseInsensitive: true, } @@ -233,7 +233,7 @@ func TestDNSMakeResult(t *testing.T) { Matchers: []*matchers.Matcher{{ Name: "test", Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, }}, Extractors: []*extractors.Extractor{{ diff --git a/v2/pkg/protocols/dns/request_test.go b/v2/pkg/protocols/dns/request_test.go index 138fee6f..7b6a3e6a 100644 --- a/v2/pkg/protocols/dns/request_test.go +++ b/v2/pkg/protocols/dns/request_test.go @@ -30,7 +30,7 @@ func TestDNSExecuteWithResults(t *testing.T) { Matchers: []*matchers.Matcher{{ Name: "test", Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"93.184.216.34"}, }}, Extractors: []*extractors.Extractor{{ diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index 1f0b6da6..ac41a940 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -66,7 +66,7 @@ func TestFileOperatorMatch(t *testing.T) { t.Run("valid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, } err = matcher.CompileMatchers() @@ -80,7 +80,7 @@ func TestFileOperatorMatch(t *testing.T) { t.Run("negative", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Negative: true, Words: []string{"random"}, } @@ -95,7 +95,7 @@ func TestFileOperatorMatch(t *testing.T) { t.Run("invalid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"random"}, } err := matcher.CompileMatchers() @@ -114,7 +114,7 @@ func TestFileOperatorMatch(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"TeSt-DaTA"}, CaseInsensitive: true, } @@ -200,13 +200,13 @@ func testFileMakeResultOperators(t *testing.T, matcherCondition string) *output. matcher := []*matchers.Matcher{ { Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: expectedValue, }, { Name: namedMatcherName, Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: expectedValue, }, } diff --git a/v2/pkg/protocols/file/request_test.go b/v2/pkg/protocols/file/request_test.go index fa0c8120..ba057daf 100644 --- a/v2/pkg/protocols/file/request_test.go +++ b/v2/pkg/protocols/file/request_test.go @@ -32,7 +32,7 @@ func TestFileExecuteWithResults(t *testing.T) { Matchers: []*matchers.Matcher{{ Name: "test", Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, }}, Extractors: []*extractors.Extractor{{ diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index 50e9c9d0..7aefd7d1 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -78,7 +78,7 @@ func TestHTTPOperatorMatch(t *testing.T) { t.Run("valid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, } err = matcher.CompileMatchers() @@ -92,7 +92,7 @@ func TestHTTPOperatorMatch(t *testing.T) { t.Run("negative", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Negative: true, Words: []string{"random"}, } @@ -107,7 +107,7 @@ func TestHTTPOperatorMatch(t *testing.T) { t.Run("invalid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"random"}, } err := matcher.CompileMatchers() @@ -121,7 +121,7 @@ func TestHTTPOperatorMatch(t *testing.T) { t.Run("caseInsensitive", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", // only applies to word + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, // only applies to word Words: []string{"EXAMPLE DOMAIN"}, CaseInsensitive: true, } @@ -262,7 +262,7 @@ func TestHTTPMakeResult(t *testing.T) { Matchers: []*matchers.Matcher{{ Name: "test", Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, }}, Extractors: []*extractors.Extractor{{ diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index 292473bf..daa92725 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -64,7 +64,7 @@ func TestNetworkOperatorMatch(t *testing.T) { t.Run("valid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"STAT "}, } err = matcher.CompileMatchers() @@ -78,7 +78,7 @@ func TestNetworkOperatorMatch(t *testing.T) { t.Run("negative", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "data", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Negative: true, Words: []string{"random"}, } @@ -93,7 +93,7 @@ func TestNetworkOperatorMatch(t *testing.T) { t.Run("invalid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "data", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"random"}, } err := matcher.CompileMatchers() @@ -107,7 +107,7 @@ func TestNetworkOperatorMatch(t *testing.T) { t.Run("caseInsensitive", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"rESp-DAta"}, CaseInsensitive: true, } @@ -188,7 +188,7 @@ func TestNetworkMakeResult(t *testing.T) { Matchers: []*matchers.Matcher{{ Name: "test", Part: "data", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"STAT "}, }}, Extractors: []*extractors.Extractor{{ diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index 1e51b9c6..3cbff4d9 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -33,7 +33,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { Matchers: []*matchers.Matcher{{ Name: "test", Part: "data", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"200 OK"}, }}, Extractors: []*extractors.Extractor{{ diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index 1290dfb7..b69ccbdc 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -70,7 +70,7 @@ func TestHTTPOperatorMatch(t *testing.T) { t.Run("valid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, } err = matcher.CompileMatchers() @@ -84,7 +84,7 @@ func TestHTTPOperatorMatch(t *testing.T) { t.Run("negative", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Negative: true, Words: []string{"random"}, } @@ -99,7 +99,7 @@ func TestHTTPOperatorMatch(t *testing.T) { t.Run("invalid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"random"}, } err := matcher.CompileMatchers() @@ -179,7 +179,7 @@ func TestHTTPMakeResult(t *testing.T) { Matchers: []*matchers.Matcher{{ Name: "test", Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, }}, Extractors: []*extractors.Extractor{{ diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index c2976657..49e58eb0 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -29,9 +29,9 @@ var ( Operators: operators.Operators{ MatchersCondition: "and", Matchers: []*matchers.Matcher{ - {Type: "word", Words: []string{"[core]"}}, - {Type: "dsl", DSL: []string{"!contains(tolower(body), ' Date: Fri, 19 Nov 2021 10:55:27 +0000 Subject: [PATCH 136/146] Auto Generate Syntax Docs + JSONSchema [Fri Nov 19 10:55:27 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 18 +++++++++--------- nuclei-jsonschema.json | 24 +++++++++++++++--------- v2/pkg/templates/templates_doc.go | 2 +- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 6ceae433..c3fb0b63 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -87,15 +87,15 @@ Examples: ```yaml requests: matchers: - - type: word + - type: 1 words: - '[core]' - - type: dsl + - type: 6 condition: and dsl: - '!contains(tolower(body), '' -type string +type MatcherTypeHolder
@@ -2396,7 +2396,7 @@ inputs: - data: "envi\r\nquit\r\n" read-size: 2048 matchers: - - type: word + - type: 1 words: - zookeeper.version ``` diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index ddaf889f..32d88583 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -211,15 +211,8 @@ ], "properties": { "type": { - "enum": [ - "status", - "size", - "word", - "regex", - "binary", - "dsl" - ], - "type": "string", + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/matchers.MatcherTypeHolder", "title": "type of matcher", "description": "Type of the matcher" }, @@ -312,6 +305,19 @@ "additionalProperties": false, "type": "object" }, + "matchers.MatcherTypeHolder": { + "enum": [ + 1, + 2, + 3, + 4, + 5, + 6 + ], + "type": "string", + "title": "type of the matcher", + "description": "Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl" + }, "generators.AttackTypeHolder": { "enum": [ "batteringram", diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index df4ff0a4..eb547d0a 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -497,7 +497,7 @@ func init() { } MATCHERSMatcherDoc.Fields = make([]encoder.Doc, 13) MATCHERSMatcherDoc.Fields[0].Name = "type" - MATCHERSMatcherDoc.Fields[0].Type = "string" + MATCHERSMatcherDoc.Fields[0].Type = "MatcherTypeHolder" MATCHERSMatcherDoc.Fields[0].Note = "" MATCHERSMatcherDoc.Fields[0].Description = "Type is the type of the matcher." MATCHERSMatcherDoc.Fields[0].Comments[encoder.LineComment] = "Type is the type of the matcher." From f74ff3fc496c7e5b6980cd9bf0fe740ef64b02e5 Mon Sep 17 00:00:00 2001 From: Sajad Date: Sat, 20 Nov 2021 13:25:27 +0530 Subject: [PATCH 137/146] Tag based struct validation (#1256) * Added tag based struct validation --- .../test-issue-tracker-config1.yaml | 4 +- .../test-issue-tracker-config2.yaml | 4 +- v2/go.mod | 5 +++ v2/go.sum | 14 +++++++ v2/internal/runner/options.go | 15 ++++++- v2/internal/runner/runner.go | 6 +-- .../reporting/exporters/es/elasticsearch.go | 40 +++---------------- v2/pkg/reporting/trackers/github/github.go | 36 +++-------------- v2/pkg/reporting/trackers/gitlab/gitlab.go | 34 ++-------------- v2/pkg/reporting/trackers/jira/jira.go | 40 +++---------------- v2/pkg/types/types.go | 2 +- v2/pkg/utils/yaml/yaml_decode_wrapper.go | 34 ++++++++++++++++ 12 files changed, 94 insertions(+), 140 deletions(-) create mode 100644 v2/pkg/utils/yaml/yaml_decode_wrapper.go diff --git a/integration_tests/test-issue-tracker-config1.yaml b/integration_tests/test-issue-tracker-config1.yaml index b7c1f73c..dd8ceb18 100644 --- a/integration_tests/test-issue-tracker-config1.yaml +++ b/integration_tests/test-issue-tracker-config1.yaml @@ -26,8 +26,8 @@ gitlab: username: test-username # token is the token for gitlab account. token: test-token - # project-id is the ID of the repository. - project-id: 1234 + # project-name is the name/id of the project(repository). + project-name: "1234" # issue-label is the label of the created issue type issue-label: bug diff --git a/integration_tests/test-issue-tracker-config2.yaml b/integration_tests/test-issue-tracker-config2.yaml index eeb6eaa3..c76b773e 100644 --- a/integration_tests/test-issue-tracker-config2.yaml +++ b/integration_tests/test-issue-tracker-config2.yaml @@ -28,8 +28,8 @@ gitlab: username: test-username # token is the token for gitlab account. token: test-token - # project-id is the ID of the repository. - project-id: 1234 + # project-name is the name/id of the project(repository). + project-name: "1234" # issue-label is the label of the created issue type issue-label: bug diff --git a/v2/go.mod b/v2/go.mod index c7d51d01..74eb7ea3 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -12,6 +12,7 @@ require ( github.com/blang/semver v3.5.1+incompatible github.com/bluele/gcache v0.0.2 github.com/corpix/uarand v0.1.1 + github.com/go-playground/validator/v10 v10.9.0 github.com/go-rod/rod v0.101.8 github.com/gobwas/ws v1.1.0 github.com/google/go-github v17.0.0+incompatible @@ -82,6 +83,8 @@ require ( github.com/eggsampler/acme/v3 v3.2.1 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/go-ole/go-ole v1.2.5 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect @@ -100,6 +103,7 @@ require ( github.com/karlseguin/ccache/v2 v2.0.8 // indirect github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/pgzip v1.2.5 // indirect + github.com/leodido/go-urn v1.2.1 // indirect github.com/mattn/go-isatty v0.0.13 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -119,6 +123,7 @@ require ( github.com/ysmood/goob v0.3.0 // indirect github.com/zclconf/go-cty v1.8.4 // indirect go.etcd.io/bbolt v1.3.6 // indirect + golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index c649ceb4..f474a727 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -230,6 +230,14 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/validator/v10 v10.9.0 h1:NgTtmN58D0m8+UuxtYmGztBJB7VnPgjj221I1QHci2A= +github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI= github.com/go-rod/rod v0.101.8 h1:oV0O97uwjkCVyAP0hD6K6bBE8FUMIjs0dtF7l6kEBsU= @@ -456,6 +464,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= @@ -841,6 +851,8 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1014,8 +1026,10 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index b4dc2f3d..f11d2055 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -2,11 +2,13 @@ package runner import ( "bufio" - "errors" "os" "path/filepath" "strings" + "github.com/pkg/errors" + + "github.com/go-playground/validator/v10" "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" @@ -77,6 +79,17 @@ func hasStdin() bool { // validateOptions validates the configuration options passed func validateOptions(options *types.Options) error { + validate := validator.New() + if err := validate.Struct(options); err != nil { + if _, ok := err.(*validator.InvalidValidationError); ok { + return err + } + errs := []string{} + for _, err := range err.(validator.ValidationErrors) { + errs = append(errs, err.Namespace()+": "+err.Tag()) + } + return errors.Wrap(errors.New(strings.Join(errs, ", ")), "validation failed for these fields") + } if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 5e71b76f..9e9ee8f1 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -10,7 +10,6 @@ import ( "github.com/logrusorgru/aurora" "github.com/pkg/errors" "go.uber.org/ratelimit" - "gopkg.in/yaml.v2" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/colorizer" @@ -35,6 +34,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" + yamlwrapper "github.com/projectdiscovery/nuclei/v2/pkg/utils/yaml" ) // Runner is a client for running the enumeration process. @@ -180,9 +180,9 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error) } reportingOptions = &reporting.Options{} - if parseErr := yaml.NewDecoder(file).Decode(reportingOptions); parseErr != nil { + if err := yamlwrapper.DecodeAndValidate(file, reportingOptions); err != nil { file.Close() - return nil, errors.Wrap(parseErr, "could not parse reporting config file") + return nil, errors.Wrap(err, "could not parse reporting config file") } file.Close() } diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index 784e587d..0959cc92 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -6,7 +6,6 @@ import ( "fmt" "io/ioutil" "net/http" - "strings" "time" "encoding/base64" @@ -20,19 +19,19 @@ import ( // Options contains necessary options required for elasticsearch communicaiton type Options struct { // IP for elasticsearch instance - IP string `yaml:"ip"` + IP string `yaml:"ip" validate:"required,ip"` // Port is the port of elasticsearch instance - Port int `yaml:"port"` + Port int `yaml:"port" validate:"required,gte=0,lte=65535"` // SSL (optional) enables ssl for elasticsearch connection SSL bool `yaml:"ssl"` // SSLVerification (optional) disables SSL verification for elasticsearch SSLVerification bool `yaml:"ssl-verification"` // Username for the elasticsearch instance - Username string `yaml:"username"` + Username string `yaml:"username" validate:"required"` // Password is the password for elasticsearch instance - Password string `yaml:"password"` + Password string `yaml:"password" validate:"required"` // IndexName is the name of the elasticsearch index - IndexName string `yaml:"index-name"` + IndexName string `yaml:"index-name" validate:"required"` } type data struct { @@ -50,10 +49,6 @@ type Exporter struct { // New creates and returns a new exporter for elasticsearch func New(option *Options) (*Exporter, error) { var ei *Exporter - err := validateOptions(option) - if err != nil { - return nil, err - } client := &http.Client{ Timeout: 5 * time.Second, @@ -86,31 +81,6 @@ func New(option *Options) (*Exporter, error) { return ei, nil } -func validateOptions(options *Options) error { - errs := []string{} - if options.IP == "" { - errs = append(errs, "IP") - } - if options.Port == 0 { - errs = append(errs, "Port") - } - if options.Username == "" { - errs = append(errs, "Username") - } - if options.Password == "" { - errs = append(errs, "Password") - } - if options.IndexName == "" { - errs = append(errs, "IndexName") - } - - if len(errs) > 0 { - return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) - } - - return nil -} - // Export exports a passed result event to elasticsearch func (i *Exporter) Export(event *output.ResultEvent) error { // creating a request diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index 8d6cdb86..47225ada 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -24,15 +24,15 @@ type Integration struct { // Options contains the configuration options for github issue tracker client type Options struct { // BaseURL (optional) is the self-hosted github application url - BaseURL string `yaml:"base-url"` + BaseURL string `yaml:"base-url" validate:"omitempty,url"` // Username is the username of the github user - Username string `yaml:"username"` + Username string `yaml:"username" validate:"required"` // Owner (manadatory) is the owner name of the repository for issues. - Owner string `yaml:"owner"` + Owner string `yaml:"owner" validate:"required"` // Token is the token for github account. - Token string `yaml:"token"` + Token string `yaml:"token" validate:"required"` // ProjectName is the name of the repository. - ProjectName string `yaml:"project-name"` + ProjectName string `yaml:"project-name" validate:"required"` // IssueLabel (optional) is the label of the created issue type IssueLabel string `yaml:"issue-label"` // SeverityAsLabel (optional) sends the severity as the label of the created @@ -42,10 +42,6 @@ type Options struct { // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { - err := validateOptions(options) - if err != nil { - return nil, err - } ctx := context.Background() ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: options.Token}, @@ -66,28 +62,6 @@ func New(options *Options) (*Integration, error) { return &Integration{client: client, options: options}, nil } -func validateOptions(options *Options) error { - errs := []string{} - if options.Username == "" { - errs = append(errs, "Username") - } - if options.Owner == "" { - errs = append(errs, "Owner") - } - if options.Token == "" { - errs = append(errs, "Token") - } - if options.ProjectName == "" { - errs = append(errs, "ProjectName") - } - - if len(errs) > 0 { - return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) - } - - return nil -} - // CreateIssue creates an issue in the tracker func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index 35922970..c1438e2d 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -2,9 +2,6 @@ package gitlab import ( "fmt" - "strings" - - "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" @@ -21,13 +18,13 @@ type Integration struct { // Options contains the configuration options for gitlab issue tracker client type Options struct { // BaseURL (optional) is the self-hosted gitlab application url - BaseURL string `yaml:"base-url"` + BaseURL string `yaml:"base-url" validate:"omitempty,url"` // Username is the username of the gitlab user - Username string `yaml:"username"` + Username string `yaml:"username" validate:"required"` // Token is the token for gitlab account. - Token string `yaml:"token"` + Token string `yaml:"token" validate:"required"` // ProjectName is the name of the repository. - ProjectName string `yaml:"project-name"` + ProjectName string `yaml:"project-name" validate:"required"` // IssueLabel is the label of the created issue type IssueLabel string `yaml:"issue-label"` // SeverityAsLabel (optional) sends the severity as the label of the created @@ -37,10 +34,6 @@ type Options struct { // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { - err := validateOptions(options) - if err != nil { - return nil, err - } gitlabOpts := []gitlab.ClientOptionFunc{} if options.BaseURL != "" { gitlabOpts = append(gitlabOpts, gitlab.WithBaseURL(options.BaseURL)) @@ -56,25 +49,6 @@ func New(options *Options) (*Integration, error) { return &Integration{client: git, userID: user.ID, options: options}, nil } -func validateOptions(options *Options) error { - errs := []string{} - if options.Username == "" { - errs = append(errs, "Username") - } - if options.Token == "" { - errs = append(errs, "Token") - } - if options.ProjectName == "" { - errs = append(errs, "ProjectName") - } - - if len(errs) > 0 { - return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) - } - - return nil -} - // CreateIssue creates an issue in the tracker func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 3237e929..16ef1b2a 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -2,7 +2,6 @@ package jira import ( "bytes" - "errors" "fmt" "io/ioutil" "strings" @@ -28,15 +27,15 @@ type Options struct { // UpdateExisting value (optional) if true, the existing opened issue is updated UpdateExisting bool `yaml:"update-existing"` // URL is the URL of the jira server - URL string `yaml:"url"` + URL string `yaml:"url" validate:"required"` // AccountID is the accountID of the jira user. - AccountID string `yaml:"account-id"` + AccountID string `yaml:"account-id" validate:"required"` // Email is the email of the user for jira instance - Email string `yaml:"email"` + Email string `yaml:"email" validate:"required,email"` // Token is the token for jira instance. - Token string `yaml:"token"` + Token string `yaml:"token" validate:"required"` // ProjectName is the name of the project. - ProjectName string `yaml:"project-name"` + ProjectName string `yaml:"project-name" validate:"required"` // IssueType (optional) is the name of the created issue type IssueType string `yaml:"issue-type"` // SeverityAsLabel (optional) sends the severity as the label of the created @@ -46,10 +45,6 @@ type Options struct { // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { - err := validateOptions(options) - if err != nil { - return nil, err - } username := options.Email if !options.Cloud { username = options.AccountID @@ -65,31 +60,6 @@ func New(options *Options) (*Integration, error) { return &Integration{jira: jiraClient, options: options}, nil } -func validateOptions(options *Options) error { - errs := []string{} - if options.URL == "" { - errs = append(errs, "URL") - } - if options.AccountID == "" { - errs = append(errs, "AccountID") - } - if options.Email == "" { - errs = append(errs, "Email") - } - if options.Token == "" { - errs = append(errs, "Token") - } - if options.ProjectName == "" { - errs = append(errs, "ProjectName") - } - - if len(errs) > 0 { - return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) - } - - return nil -} - // CreateNewIssue creates a new issue in the tracker func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { summary := format.Summary(event) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 65958c7a..c2619b7b 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -49,7 +49,7 @@ type Options struct { // ProjectPath allows nuclei to use a user defined project folder ProjectPath string // InteractshURL is the URL for the interactsh server. - InteractshURL string + InteractshURL string `validate:"omitempty,url"` // Interactsh Authorization header value for self-hosted servers InteractshToken string // Target URLs/Domains to scan using a template diff --git a/v2/pkg/utils/yaml/yaml_decode_wrapper.go b/v2/pkg/utils/yaml/yaml_decode_wrapper.go new file mode 100644 index 00000000..a9cb422b --- /dev/null +++ b/v2/pkg/utils/yaml/yaml_decode_wrapper.go @@ -0,0 +1,34 @@ +package yaml + +import ( + "io" + "strings" + + "github.com/go-playground/validator/v10" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +var validate *validator.Validate + +// DecodeAndValidate is a wrapper for yaml Decode adding struct validation +func DecodeAndValidate(r io.Reader, v interface{}) error { + if err := yaml.NewDecoder(r).Decode(v); err != nil { + return err + } + if validate == nil { + validate = validator.New() + } + if err := validate.Struct(v); err != nil { + + if _, ok := err.(*validator.InvalidValidationError); ok { + return err + } + errs := []string{} + for _, err := range err.(validator.ValidationErrors) { + errs = append(errs, err.Namespace()+": "+err.Tag()) + } + return errors.Wrap(errors.New(strings.Join(errs, ", ")), "validation failed for these fields") + } + return nil +} From 66dacccfb481f3f0bba958b8aee917fc9f5981e1 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 20 Nov 2021 19:47:05 +0530 Subject: [PATCH 138/146] Fixed matcher type integer issue --- v2/pkg/operators/matchers/matchers_types.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/operators/matchers/matchers_types.go b/v2/pkg/operators/matchers/matchers_types.go index 312d4f59..f307d08b 100644 --- a/v2/pkg/operators/matchers/matchers_types.go +++ b/v2/pkg/operators/matchers/matchers_types.go @@ -106,10 +106,10 @@ func (holder *MatcherTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error return nil } -func (holder *MatcherTypeHolder) MarshalJSON() ([]byte, error) { - return json.Marshal(holder.MatcherType) +func (holder MatcherTypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.MatcherType.String()) } func (holder MatcherTypeHolder) MarshalYAML() (interface{}, error) { - return holder.MatcherType, nil + return holder.MatcherType.String(), nil } From 925fb18855a2ab6de3b5b5565d03d7d6a59c6291 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sat, 20 Nov 2021 14:18:20 +0000 Subject: [PATCH 139/146] Auto Generate Syntax Docs + JSONSchema [Sat Nov 20 14:18:20 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index c3fb0b63..6c174243 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -87,15 +87,15 @@ Examples: ```yaml requests: matchers: - - type: 1 + - type: word words: - '[core]' - - type: 6 + - type: dsl condition: and dsl: - '!contains(tolower(body), '' Date: Sat, 20 Nov 2021 19:50:18 +0530 Subject: [PATCH 140/146] Fixed jsonschema for matchertype missing String() --- SYNTAX-REFERENCE.md | 16 ++++++++-------- nuclei-jsonschema.json | 12 ++++++------ v2/pkg/operators/matchers/matchers_types.go | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index c3fb0b63..6c174243 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -87,15 +87,15 @@ Examples: ```yaml requests: matchers: - - type: 1 + - type: word words: - '[core]' - - type: 6 + - type: dsl condition: and dsl: - '!contains(tolower(body), '' Date: Mon, 22 Nov 2021 00:19:53 +0100 Subject: [PATCH 141/146] Adding support for clustering within workflow (#1255) * Adding support for clustering within workflow --- DESIGN.md | 3 +- v2/internal/runner/runner.go | 3 +- v2/pkg/core/execute.go | 40 +-------- .../protocols/common/clusterer/clusterer.go | 49 ----------- .../executer.go => templates/cluster.go} | 85 ++++++++++++++++++- v2/pkg/templates/workflows.go | 9 ++ 6 files changed, 95 insertions(+), 94 deletions(-) delete mode 100644 v2/pkg/protocols/common/clusterer/clusterer.go rename v2/pkg/{protocols/common/clusterer/executer.go => templates/cluster.go} (62%) diff --git a/DESIGN.md b/DESIGN.md index f9875b0b..80785273 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -104,7 +104,7 @@ The `ExecuteWithResults` function accepts a callback, which gets provided with r The default executer is provided in `pkg/protocols/common/executer` . It takes a list of Requests and relevant `ExecuterOptions` and implements the Executer interface required for template execution. The executer during Template compilation process is created from this package and used as-is. -A different executer is the Clustered Requests executer which implements the Nuclei Request clustering functionality in `pkg/protocols/common/clusterer` We have a single HTTP request in cases where multiple templates can be clustered and multiple operator lists to match/extract. The first HTTP request is executed while all the template matcher/extractor are evaluated separately. +A different executer is the Clustered Requests executer which implements the Nuclei Request clustering functionality in `pkg/templates` We have a single HTTP request in cases where multiple templates can be clustered and multiple operator lists to match/extract. The first HTTP request is executed while all the template matcher/extractor are evaluated separately. For Workflow execution, a separate RunWorkflow function is used which executes the workflow independently from the template execution. @@ -585,7 +585,6 @@ That's it, you've added a new protocol to Nuclei. The next good step would be to - [v2/pkg/protocols/common/generators](./v2/pkg/protocols/common/generators) - Payload support for Requests (Sniper, etc) - [v2/pkg/protocols/common/executer](./v2/pkg/protocols/common/executer) - Default Template Executer - [v2/pkg/protocols/common/replacer](./v2/pkg/protocols/common/replacer) - Template replacement helpers -- [v2/pkg/protocols/common/clusterer](./v2/pkg/protocols/common/clusterer) - HTTP Request Clustering Implementation - [v2/pkg/protocols/common/helpers/eventcreator](./v2/pkg/protocols/common/helpers/eventcreator) - Result event creator - [v2/pkg/protocols/common/helpers/responsehighlighter](./v2/pkg/protocols/common/helpers/responsehighlighter) - Debug response highlighter - [v2/pkg/protocols/common/helpers/deserialization](./v2/pkg/protocols/common/helpers/deserialization) - Deserialization helper functions diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 9e9ee8f1..9a1d6e71 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -31,6 +31,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/reporting" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" @@ -306,7 +307,7 @@ func (r *Runner) RunEnumeration() error { // Cluster the templates first because we want info on how many // templates did we cluster for showing to user in CLI originalTemplatesCount := len(store.Templates()) - finalTemplates, clusterCount := engine.ClusterTemplates(store.Templates()) + finalTemplates, clusterCount := templates.ClusterTemplates(store.Templates(), engine.ExecuterOptions()) finalTemplates = append(finalTemplates, store.Workflows()...) var totalRequests int64 diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index c9d36cd9..f7840010 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -1,14 +1,10 @@ package core import ( - "fmt" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/remeh/sizedwaitgroup" - "github.com/rs/xid" "go.uber.org/atomic" ) @@ -25,7 +21,7 @@ func (e *Engine) Execute(templates []*templates.Template, target InputProvider) func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target InputProvider, noCluster bool) *atomic.Bool { var finalTemplates []*templates.Template if !noCluster { - finalTemplates, _ = e.ClusterTemplates(templatesList) + finalTemplates, _ = templates.ClusterTemplates(templatesList, e.executerOpts) } else { finalTemplates = templatesList } @@ -95,37 +91,3 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template }) wg.Waitgroup.Wait() } - -// ClusterTemplates performs identical http requests clustering for a list of templates -func (e *Engine) ClusterTemplates(templatesList []*templates.Template) ([]*templates.Template, int) { - if e.options.OfflineHTTP { - return templatesList, 0 - } - - templatesMap := make(map[string]*templates.Template) - for _, v := range templatesList { - templatesMap[v.Path] = v - } - clusterCount := 0 - - finalTemplatesList := make([]*templates.Template, 0, len(templatesList)) - clusters := clusterer.Cluster(templatesMap) - for _, cluster := range clusters { - if len(cluster) > 1 { - executerOpts := e.ExecuterOptions() - - clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) - - finalTemplatesList = append(finalTemplatesList, &templates.Template{ - ID: clusterID, - RequestsHTTP: cluster[0].RequestsHTTP, - Executer: clusterer.NewExecuter(cluster, &executerOpts), - TotalRequests: len(cluster[0].RequestsHTTP), - }) - clusterCount += len(cluster) - } else { - finalTemplatesList = append(finalTemplatesList, cluster...) - } - } - return finalTemplatesList, clusterCount -} diff --git a/v2/pkg/protocols/common/clusterer/clusterer.go b/v2/pkg/protocols/common/clusterer/clusterer.go deleted file mode 100644 index 29c1a309..00000000 --- a/v2/pkg/protocols/common/clusterer/clusterer.go +++ /dev/null @@ -1,49 +0,0 @@ -package clusterer - -import ( - "github.com/projectdiscovery/nuclei/v2/pkg/templates" -) - -// Cluster clusters a list of templates into a lesser number if possible based -// on the similarity between the sent requests. -// -// If the attributes match, multiple requests can be clustered into a single -// request which saves time and network resources during execution. -func Cluster(list map[string]*templates.Template) [][]*templates.Template { - final := [][]*templates.Template{} - - // Each protocol that can be clustered should be handled here. - for key, template := range list { - // We only cluster http requests as of now. - // Take care of requests that can't be clustered first. - if len(template.RequestsHTTP) == 0 { - delete(list, key) - final = append(final, []*templates.Template{template}) - continue - } - - delete(list, key) // delete element first so it's not found later. - // Find any/all similar matching request that is identical to - // this one and cluster them together for http protocol only. - if len(template.RequestsHTTP) == 1 { - cluster := []*templates.Template{} - - for otherKey, other := range list { - if len(other.RequestsHTTP) == 0 { - continue - } - if template.RequestsHTTP[0].CanCluster(other.RequestsHTTP[0]) { - delete(list, otherKey) - cluster = append(cluster, other) - } - } - if len(cluster) > 0 { - cluster = append(cluster, template) - final = append(final, cluster) - continue - } - } - final = append(final, []*templates.Template{template}) - } - return final -} diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/templates/cluster.go similarity index 62% rename from v2/pkg/protocols/common/clusterer/executer.go rename to v2/pkg/templates/cluster.go index c060f341..7e0ed1cf 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/templates/cluster.go @@ -1,15 +1,94 @@ -package clusterer +package templates import ( + "fmt" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/rs/xid" ) +// Cluster clusters a list of templates into a lesser number if possible based +// on the similarity between the sent requests. +// +// If the attributes match, multiple requests can be clustered into a single +// request which saves time and network resources during execution. +func Cluster(list map[string]*Template) [][]*Template { + final := [][]*Template{} + + // Each protocol that can be clustered should be handled here. + for key, template := range list { + // We only cluster http requests as of now. + // Take care of requests that can't be clustered first. + if len(template.RequestsHTTP) == 0 { + delete(list, key) + final = append(final, []*Template{template}) + continue + } + + delete(list, key) // delete element first so it's not found later. + // Find any/all similar matching request that is identical to + // this one and cluster them together for http protocol only. + if len(template.RequestsHTTP) == 1 { + cluster := []*Template{} + + for otherKey, other := range list { + if len(other.RequestsHTTP) == 0 { + continue + } + if template.RequestsHTTP[0].CanCluster(other.RequestsHTTP[0]) { + delete(list, otherKey) + cluster = append(cluster, other) + } + } + if len(cluster) > 0 { + cluster = append(cluster, template) + final = append(final, cluster) + continue + } + } + final = append(final, []*Template{template}) + } + return final +} + +func ClusterTemplates(templatesList []*Template, options protocols.ExecuterOptions) ([]*Template, int) { + if options.Options.OfflineHTTP { + return templatesList, 0 + } + + templatesMap := make(map[string]*Template) + for _, v := range templatesList { + templatesMap[v.Path] = v + } + clusterCount := 0 + + finalTemplatesList := make([]*Template, 0, len(templatesList)) + clusters := Cluster(templatesMap) + for _, cluster := range clusters { + if len(cluster) > 1 { + executerOpts := options + + clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) + + finalTemplatesList = append(finalTemplatesList, &Template{ + ID: clusterID, + RequestsHTTP: cluster[0].RequestsHTTP, + Executer: NewExecuter(cluster, &executerOpts), + TotalRequests: len(cluster[0].RequestsHTTP), + }) + clusterCount += len(cluster) + } else { + finalTemplatesList = append(finalTemplatesList, cluster...) + } + } + return finalTemplatesList, clusterCount +} + // Executer executes a group of requests for a protocol for a clustered // request. It is different from normal executers since the original // operators are all combined and post processed after making the request. @@ -31,7 +110,7 @@ type clusteredOperator struct { var _ protocols.Executer = &Executer{} // NewExecuter creates a new request executer for list of requests -func NewExecuter(requests []*templates.Template, options *protocols.ExecuterOptions) *Executer { +func NewExecuter(requests []*Template, options *protocols.ExecuterOptions) *Executer { executer := &Executer{ options: options, requests: requests[0].RequestsHTTP[0], diff --git a/v2/pkg/templates/workflows.go b/v2/pkg/templates/workflows.go index 246c4bf8..89f8d1bd 100644 --- a/v2/pkg/templates/workflows.go +++ b/v2/pkg/templates/workflows.go @@ -61,6 +61,9 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr if len(paths) == 0 { return nil } + + var workflowTemplates []*Template + for _, path := range paths { template, err := Parse(path, preprocessor, options.Copy()) if err != nil { @@ -71,10 +74,16 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr gologger.Warning().Msgf("Could not parse workflow template %s: no executer found\n", path) continue } + workflowTemplates = append(workflowTemplates, template) + } + + finalTemplates, _ := ClusterTemplates(workflowTemplates, options.Copy()) + for _, template := range finalTemplates { workflow.Executers = append(workflow.Executers, &workflows.ProtocolExecuterPair{ Executer: template.Executer, Options: options, }) } + return nil } From ba67f8c8fec6b2bd0f939e93f691fa85373a2129 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 22 Nov 2021 09:51:13 +0530 Subject: [PATCH 142/146] fix #1220: no ip returned for dialers --- v2/pkg/protocols/common/protocolstate/state.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/protocols/common/protocolstate/state.go b/v2/pkg/protocols/common/protocolstate/state.go index 0f80f664..b43e40cf 100644 --- a/v2/pkg/protocols/common/protocolstate/state.go +++ b/v2/pkg/protocols/common/protocolstate/state.go @@ -18,6 +18,7 @@ func Init(options *types.Options) error { if options.ResolversFile != "" { opts.BaseResolvers = options.InternalResolversList } + opts.WithDialerHistory = true dialer, err := fastdialer.NewDialer(opts) if err != nil { return errors.Wrap(err, "could not create dialer") From 1581c96e4e5f90b7c7c4d606be92e6429b3daf92 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Mon, 22 Nov 2021 17:53:25 +0530 Subject: [PATCH 143/146] Added matched-status flag + template-path and url to output (#1272) * Added matched-status flag + template-path and url to output --- v2/cmd/nuclei/main.go | 1 + v2/internal/runner/runner.go | 2 +- v2/pkg/output/format_screen.go | 15 ++++++- v2/pkg/output/output.go | 41 ++++++++++++++++++- v2/pkg/output/output_test.go | 6 +-- v2/pkg/protocols/common/executer/executer.go | 22 +++++----- .../protocols/common/helpers/writer/writer.go | 35 ++++++++++++++++ .../protocols/common/interactsh/interactsh.go | 16 ++------ v2/pkg/protocols/dns/operators.go | 4 +- v2/pkg/protocols/dns/operators_test.go | 2 +- v2/pkg/protocols/file/operators.go | 4 +- v2/pkg/protocols/file/operators_test.go | 10 ++--- v2/pkg/protocols/headless/operators.go | 4 +- v2/pkg/protocols/http/operators.go | 4 +- v2/pkg/protocols/http/operators_test.go | 8 ++-- v2/pkg/protocols/http/request.go | 3 ++ v2/pkg/protocols/network/operators.go | 4 +- v2/pkg/protocols/network/operators_test.go | 2 +- v2/pkg/protocols/network/request.go | 3 ++ v2/pkg/protocols/offlinehttp/operators.go | 4 +- .../protocols/offlinehttp/operators_test.go | 8 ++-- v2/pkg/protocols/ssl/ssl.go | 4 +- v2/pkg/protocols/websocket/websocket.go | 5 ++- v2/pkg/templates/cluster.go | 25 +++++------ v2/pkg/testutils/testutils.go | 5 +++ v2/pkg/types/types.go | 2 + v2/pkg/utils/template_path.go | 32 +++++++++++++++ 27 files changed, 205 insertions(+), 66 deletions(-) create mode 100644 v2/pkg/protocols/common/helpers/writer/writer.go create mode 100644 v2/pkg/utils/template_path.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 7f24385f..11647973 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -85,6 +85,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "don't display match metadata"), flagSet.BoolVarP(&options.NoTimestamp, "no-timestamp", "nts", false, "don't display timestamp metadata in CLI output"), flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "local nuclei reporting database (always use this to persist report data)"), + flagSet.BoolVarP(&options.MatcherStatus, "matcher-status", "ms", false, "show optional match failure status"), flagSet.StringVarP(&options.MarkdownExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"), flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 9a1d6e71..055eca1a 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -117,7 +117,7 @@ func New(options *types.Options) (*Runner, error) { runner.hmapInputProvider = hmapInput // Create the output file if asked - outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.Output, options.TraceLogFile, options.ErrorLogFile) + outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.MatcherStatus, options.Output, options.TraceLogFile, options.ErrorLogFile) if err != nil { return nil, errors.Wrap(err, "could not create output file") } diff --git a/v2/pkg/output/format_screen.go b/v2/pkg/output/format_screen.go index ddab852a..2331095d 100644 --- a/v2/pkg/output/format_screen.go +++ b/v2/pkg/output/format_screen.go @@ -27,6 +27,15 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte { builder.WriteString(w.aurora.BrightGreen(output.ExtractorName).Bold().String()) } + if w.matcherStatus { + builder.WriteString("] [") + if !output.MatcherStatus { + builder.WriteString(w.aurora.Red("failed").String()) + } else { + builder.WriteString(w.aurora.Green("matched").String()) + } + } + builder.WriteString("] [") builder.WriteString(w.aurora.BrightBlue(output.Type).String()) builder.WriteString("] ") @@ -35,7 +44,11 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte { builder.WriteString(w.severityColors(output.Info.SeverityHolder.Severity)) builder.WriteString("] ") } - builder.WriteString(output.Matched) + if output.Matched != "" { + builder.WriteString(output.Matched) + } else { + builder.WriteString(output.Host) + } // If any extractors, write the results if len(output.ExtractedResults) > 0 { diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 991c9923..e78699f7 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" ) @@ -27,6 +28,8 @@ type Writer interface { Colorizer() aurora.Aurora // Write writes the event to file and/or screen. Write(*ResultEvent) error + // WriteFailure writes the optional failure event for template to file and/or screen. + WriteFailure(event InternalEvent) error // Request logs a request in the trace log Request(templateID, url, requestType string, err error) } @@ -37,6 +40,7 @@ type StandardWriter struct { jsonReqResp bool noTimestamp bool noMetadata bool + matcherStatus bool aurora aurora.Aurora outputFile io.WriteCloser traceFile io.WriteCloser @@ -54,10 +58,16 @@ type InternalWrappedEvent struct { InternalEvent InternalEvent Results []*ResultEvent OperatorsResult *operators.Result + UsesInteractsh bool } // ResultEvent is a wrapped result event for a single nuclei output. type ResultEvent struct { + // Template is the relative filename for the template + Template string `json:"template,omitempty"` + // TemplateURL is the URL of the template for the result inside the nuclei + // templates repository if it belongs to the repository. + TemplateURL string `json:"template-url,omitempty"` // TemplateID is the ID of the template for the result. TemplateID string `json:"template-id"` // TemplatePath is the path of template @@ -92,12 +102,14 @@ type ResultEvent struct { Interaction *server.Interaction `json:"interaction,omitempty"` // CURLCommand is an optional curl command to reproduce the request // Only applicable if the report is for HTTP. - CURLCommand string `json:"curl-command,omitempty"` + CURLCommand string `json:"curl-command,omitempty"` + // MatcherStatus is the status of the match + MatcherStatus bool `json:"matcher-status"` FileToIndexPosition map[string]int `json:"-"` } // NewStandardWriter creates a new output writer based on user configurations -func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, file, traceFile string, errorFile string) (*StandardWriter, error) { +func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp, MatcherStatus bool, file, traceFile string, errorFile string) (*StandardWriter, error) { auroraColorizer := aurora.NewAurora(colors) var outputFile io.WriteCloser @@ -128,6 +140,7 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, json: json, jsonReqResp: jsonReqResp, noMetadata: noMetadata, + matcherStatus: MatcherStatus, noTimestamp: noTimestamp, aurora: auroraColorizer, outputFile: outputFile, @@ -140,6 +153,10 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, // Write writes the event to file and/or screen. func (w *StandardWriter) Write(event *ResultEvent) error { + // Enrich the result event with extra metadata on the template-path and url. + if event.TemplatePath != "" { + event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath)) + } event.Timestamp = time.Now() var data []byte @@ -224,3 +241,23 @@ func (w *StandardWriter) Close() { w.errorFile.Close() } } + +// WriteFailure writes the failure event for template to file and/or screen. +func (w *StandardWriter) WriteFailure(event InternalEvent) error { + if !w.matcherStatus { + return nil + } + templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"])) + data := &ResultEvent{ + Template: templatePath, + TemplateURL: templateURL, + TemplateID: types.ToString(event["template-id"]), + TemplatePath: types.ToString(event["template-path"]), + Info: event["template-info"].(model.Info), + Type: types.ToString(event["type"]), + Host: types.ToString(event["host"]), + MatcherStatus: false, + Timestamp: time.Now(), + } + return w.Write(data) +} diff --git a/v2/pkg/output/output_test.go b/v2/pkg/output/output_test.go index 9f1d188b..1648dcc3 100644 --- a/v2/pkg/output/output_test.go +++ b/v2/pkg/output/output_test.go @@ -11,7 +11,7 @@ import ( func TestStandardWriterRequest(t *testing.T) { t.Run("WithoutTraceAndError", func(t *testing.T) { - w, err := NewStandardWriter(false, false, false, false, false, "", "", "") + w, err := NewStandardWriter(false, false, false, false, false, false, "", "", "") require.NoError(t, err) require.NotPanics(t, func() { w.Request("path", "input", "http", nil) @@ -23,7 +23,7 @@ func TestStandardWriterRequest(t *testing.T) { traceWriter := &testWriteCloser{} errorWriter := &testWriteCloser{} - w, err := NewStandardWriter(false, false, false, false, false, "", "", "") + w, err := NewStandardWriter(false, false, false, false, false, false, "", "", "") w.traceFile = traceWriter w.errorFile = errorWriter require.NoError(t, err) @@ -36,7 +36,7 @@ func TestStandardWriterRequest(t *testing.T) { t.Run("ErrorWithWrappedError", func(t *testing.T) { errorWriter := &testWriteCloser{} - w, err := NewStandardWriter(false, false, false, false, false, "", "", "") + w, err := NewStandardWriter(false, false, false, false, false, false, "", "", "") w.errorFile = errorWriter require.NoError(t, err) w.Request( diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 3aaf774c..fe650e94 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -6,6 +6,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" ) // Executer executes a group of requests for a protocol @@ -59,18 +60,17 @@ func (e *Executer) Execute(input string) (bool, error) { builder.Reset() } } - if event.OperatorsResult == nil { - return - } - for _, result := range event.Results { - if e.options.IssuesClient != nil { - if err := e.options.IssuesClient.CreateIssue(result); err != nil { - gologger.Warning().Msgf("Could not create issue on tracker: %s", err) - } + // If no results were found, and also interactsh is not being used + // in that case we can skip it, otherwise we've to show failure in + // case of matcher-status flag. + if event.OperatorsResult == nil && !event.UsesInteractsh { + if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { + gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) + } + } else { + if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { + results = true } - results = true - _ = e.options.Output.Write(result) - e.options.Progress.IncrementMatched() } }) if err != nil { diff --git a/v2/pkg/protocols/common/helpers/writer/writer.go b/v2/pkg/protocols/common/helpers/writer/writer.go new file mode 100644 index 00000000..91b98f33 --- /dev/null +++ b/v2/pkg/protocols/common/helpers/writer/writer.go @@ -0,0 +1,35 @@ +package writer + +import ( + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting" +) + +// WriteResult is a helper for writing results to the output +func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progress progress.Progress, issuesClient *reporting.Client) bool { + // Handle the case where no result found for the template. + // In this case, we just show misc information about the failed + // match for the template. + if data.OperatorsResult == nil { + return false + } + var matched bool + for _, result := range data.Results { + if err := output.Write(result); err != nil { + gologger.Warning().Msgf("Could not write output event: %s\n", err) + } + if !matched { + matched = true + } + progress.IncrementMatched() + + if issuesClient != nil { + if err := issuesClient.CreateIssue(result); err != nil { + gologger.Warning().Msgf("Could not create issue on tracker: %s", err) + } + } + } + return matched +} diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 7c3e02f8..b810c346 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -19,6 +19,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" ) @@ -177,19 +178,8 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d } data.Event.Results = data.MakeResultFunc(data.Event) - for _, result := range data.Event.Results { - result.Interaction = interaction - _ = c.options.Output.Write(result) - if !c.matched { - c.matched = true - } - c.options.Progress.IncrementMatched() - - if c.options.IssuesClient != nil { - if err := c.options.IssuesClient.CreateIssue(result); err != nil { - gologger.Warning().Msgf("Could not create issue on tracker: %s", err) - } - } + if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) { + c.matched = true } return true } diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 12de136a..0c1ff929 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -90,6 +90,7 @@ func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched strin "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, + "type": request.Type().String(), "trace": traceToString(tracedata, false), } } @@ -104,10 +105,11 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "dns", + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + MatcherStatus: true, Timestamp: time.Now(), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["raw"]), diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index aef43ccf..5651ea42 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -45,7 +45,7 @@ func TestResponseToDSLMap(t *testing.T) { resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}}) event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", nil) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, dns.RcodeSuccess, event["rcode"], "could not get correct rcode") } diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index cde2e63e..41ef7fd3 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -73,6 +73,7 @@ func (request *Request) responseToDSLMap(raw, inputFilePath, matchedFileName str "path": inputFilePath, "matched": matchedFileName, "raw": raw, + "type": request.Type().String(), "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, @@ -120,10 +121,11 @@ func (request *Request) GetCompiledOperators() []*operators.Operators { func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ + MatcherStatus: true, TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "file", + Type: types.ToString(wrapped.InternalEvent["type"]), Path: types.ToString(wrapped.InternalEvent["path"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), Host: types.ToString(wrapped.InternalEvent["host"]), diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index ac41a940..ffdddc08 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -35,7 +35,7 @@ func TestResponseToDSLMap(t *testing.T) { resp := "test-data\r\n" event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") - require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, resp, event["raw"], "could not get correct resp") } @@ -60,7 +60,7 @@ func TestFileOperatorMatch(t *testing.T) { resp := "test-data\r\n1.1.1.1\r\n" event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") - require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, resp, event["raw"], "could not get correct resp") t.Run("valid", func(t *testing.T) { @@ -109,7 +109,7 @@ func TestFileOperatorMatch(t *testing.T) { t.Run("caseInsensitive", func(t *testing.T) { resp := "TEST-DATA\r\n1.1.1.1\r\n" event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") - require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, resp, event["raw"], "could not get correct resp") matcher := &matchers.Matcher{ @@ -148,7 +148,7 @@ func TestFileOperatorExtract(t *testing.T) { resp := "test-data\r\n1.1.1.1\r\n" event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") - require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, resp, event["raw"], "could not get correct resp") t.Run("extract", func(t *testing.T) { @@ -266,7 +266,7 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi fileContent := "test-data\r\n1.1.1.1\r\n" event := request.responseToDSLMap(fileContent, "/tmp", matchedFileName) - require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, fileContent, event["raw"], "could not get correct resp") finalEvent := &output.InternalWrappedEvent{InternalEvent: event} diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index 3d62e61b..b9f9b4cc 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -72,6 +72,7 @@ func (request *Request) responseToDSLMap(resp, req, host, matched string) output "matched": matched, "req": req, "data": resp, + "type": request.Type().String(), "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, @@ -92,11 +93,12 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "headless", + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["data"]), diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 868fd011..1f319ccf 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -113,6 +113,7 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw data[k] = strings.Join(v, " ") } data["host"] = host + data["type"] = request.Type().String() data["matched"] = matched data["request"] = rawReq data["response"] = rawResp @@ -141,12 +142,13 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "http", + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["response"]), diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index 7aefd7d1..ae6c644c 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -41,7 +41,7 @@ func TestResponseToDSLMap(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") } @@ -71,7 +71,7 @@ func TestHTTPOperatorMatch(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") @@ -159,7 +159,7 @@ func TestHTTPOperatorExtract(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test_header"], "could not get correct resp for header") @@ -286,7 +286,7 @@ func TestHTTPMakeResult(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 94782c2a..8f204078 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -473,6 +473,9 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta }) + if hasInteractMarkers { + event.UsesInteractsh = true + } responseContentType := resp.Header.Get("Content-Type") dumpResponse(event, request.options, response.fullResponse, formedURL, responseContentType) diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index c829b8ad..010b11d6 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -73,6 +73,7 @@ func (request *Request) responseToDSLMap(req, resp, raw, host, matched string) o "request": req, "data": resp, // Data is the last bytes read "raw": raw, // Raw is the full transaction data for network + "type": request.Type().String(), "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, @@ -93,12 +94,13 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "network", + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Metadata: wrapped.OperatorsResult.PayloadValues, Timestamp: time.Now(), + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["data"]), diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index daa92725..805c0ed2 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -35,7 +35,7 @@ func TestResponseToDSLMap(t *testing.T) { req := "test-data\r\n" resp := "resp-data\r\n" event := request.responseToDSLMap(req, resp, "test", "one.one.one.one", "one.one.one.one") - require.Len(t, event, 8, "could not get correct number of items in dsl map") + require.Len(t, event, 9, "could not get correct number of items in dsl map") require.Equal(t, resp, event["data"], "could not get correct resp") } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index ae63dc0d..35da81d1 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -281,6 +281,9 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input ExtractFunc: request.Extract, }) } + if len(interactshURLs) > 0 { + event.UsesInteractsh = true + } dumpResponse(event, request.options, response, actualAddress) diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index b5141037..20369309 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -113,6 +113,7 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw data["content_length"] = resp.ContentLength data["status_code"] = resp.StatusCode data["body"] = body + data["type"] = request.Type().String() data["all_headers"] = headers data["duration"] = duration.Seconds() data["template-id"] = request.options.TemplateID @@ -135,11 +136,12 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "http", + Type: types.ToString(wrapped.InternalEvent["type"]), Path: types.ToString(wrapped.InternalEvent["path"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["raw"]), diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index b69ccbdc..0a3a663f 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -37,7 +37,7 @@ func TestResponseToDSLMap(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") } @@ -63,7 +63,7 @@ func TestHTTPOperatorMatch(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") @@ -132,7 +132,7 @@ func TestHTTPOperatorExtract(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test-header"], "could not get correct resp for header") @@ -198,7 +198,7 @@ func TestHTTPMakeResult(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index ba26c8e3..388886be 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -129,6 +129,7 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous data := make(map[string]interface{}) cert := connTLS.ConnectionState().PeerCertificates[0] + data["type"] = request.Type().String() data["response"] = jsonDataString data["host"] = input data["matched"] = addressToDial @@ -195,12 +196,13 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, - Type: request.Type().String(), + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["host"]), Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), } return data diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 47a87a78..ab864f65 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -248,6 +248,8 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam for k, v := range events { data[k] = v } + + data["type"] = request.Type().String() data["success"] = "true" data["request"] = requestOutput data["response"] = responseBuilder.String() @@ -364,12 +366,13 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, - Type: request.Type().String(), + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["response"]), diff --git a/v2/pkg/templates/cluster.go b/v2/pkg/templates/cluster.go index 7e0ed1cf..1ec1e95c 100644 --- a/v2/pkg/templates/cluster.go +++ b/v2/pkg/templates/cluster.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/rs/xid" ) @@ -147,22 +148,22 @@ func (e *Executer) Execute(input string) (bool, error) { err := e.requests.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) + event.InternalEvent["template-id"] = operator.templateID + event.InternalEvent["template-path"] = operator.templatePath + event.InternalEvent["template-info"] = operator.templateInfo + + if result == nil && !matched { + if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { + gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) + } + continue + } if matched && result != nil { event.OperatorsResult = result - event.InternalEvent["template-id"] = operator.templateID - event.InternalEvent["template-path"] = operator.templatePath - event.InternalEvent["template-info"] = operator.templateInfo event.Results = e.requests.MakeResultEvent(event) results = true - for _, r := range event.Results { - if e.options.IssuesClient != nil { - if err := e.options.IssuesClient.CreateIssue(r); err != nil { - gologger.Warning().Msgf("Could not create issue on tracker: %s", err) - } - } - _ = e.options.Output.Write(r) - e.options.Progress.IncrementMatched() - } + + _ = writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) } } }) diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index 8e0c5923..982fbabc 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -131,6 +131,11 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro } } +// Write writes the event to file and/or screen. +func (m *MockOutputWriter) WriteFailure(result output.InternalEvent) error { + return nil +} + type MockProgressClient struct{} // Stop stops the progress recorder. diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index c2619b7b..1a093533 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -177,6 +177,8 @@ type Options struct { NoUpdateTemplates bool // EnvironmentVariables enables support for environment variables EnvironmentVariables bool + // MatcherStatus displays optional status for the failed matches as well + MatcherStatus bool // ClientCertFile client certificate file (PEM-encoded) used for authenticating against scanned hosts ClientCertFile string // ClientKeyFile client key file (PEM-encoded) used for authenticating against scanned hosts diff --git a/v2/pkg/utils/template_path.go b/v2/pkg/utils/template_path.go new file mode 100644 index 00000000..2039c2d9 --- /dev/null +++ b/v2/pkg/utils/template_path.go @@ -0,0 +1,32 @@ +package utils + +import ( + "strings" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" +) + +const ( + // TemplatesRepoURL is the URL for files in nuclei-templates repository + TemplatesRepoURL = "https://github.com/projectdiscovery/nuclei-templates/blob/master/" +) + +var configData *config.Config + +func init() { + configData, _ = config.ReadConfiguration() +} + +// TemplatePathURL returns the Path and URL for the provided template +func TemplatePathURL(fullPath string) (string, string) { + var templateDirectory string + if configData != nil && configData.TemplatesDirectory != "" && strings.HasPrefix(fullPath, configData.TemplatesDirectory) { + templateDirectory = configData.TemplatesDirectory + } else { + return "", "" + } + + finalPath := strings.TrimPrefix(strings.TrimPrefix(fullPath, templateDirectory), "/") + templateURL := TemplatesRepoURL + finalPath + return finalPath, templateURL +} From dfe284664ce1f334a9a5baf04e925264ab62f219 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Fri, 26 Nov 2021 18:51:02 +0530 Subject: [PATCH 144/146] Fixed a crash with http module (#1285) --- v2/pkg/protocols/http/request.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 8f204078..a0bce62e 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -405,6 +405,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } var dumpedResponse []redirectedResponse + var gotData []byte // If the status code is HTTP 101, we should not proceed with reading body. if resp.StatusCode != http.StatusSwitchingProtocols { var bodyReader io.Reader @@ -422,6 +423,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate return errors.Wrap(err, "could not read http body") } } + gotData = data resp.Body.Close() dumpedResponse, err = dumpResponseWithRedirectChain(resp, data) @@ -432,14 +434,17 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate dumpedResponse = []redirectedResponse{{fullResponse: dumpedResponseHeaders, headers: dumpedResponseHeaders}} } - for _, response := range dumpedResponse { - // if nuclei-project is enabled store the response if not previously done - if request.options.ProjectFile != nil && !fromCache { - if err := request.options.ProjectFile.Set(dumpedRequest, resp, response.body); err != nil { - return errors.Wrap(err, "could not store in project file") - } + // if nuclei-project is enabled store the response if not previously done + if request.options.ProjectFile != nil && !fromCache { + if err := request.options.ProjectFile.Set(dumpedRequest, resp, gotData); err != nil { + return errors.Wrap(err, "could not store in project file") } + } + for _, response := range dumpedResponse { + if response.resp == nil { + continue // Skip nil responses + } matchedURL := reqURL if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" { matchedURL = generatedRequest.rawRequest.FullURL From b59582bde7188932cd9776c5433e43eff4caba4c Mon Sep 17 00:00:00 2001 From: Ice3man Date: Sun, 28 Nov 2021 04:31:39 +0530 Subject: [PATCH 145/146] Fixed stdin input parsing bug (#1286) --- v2/internal/runner/options.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index f11d2055..c382f69d 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -71,9 +71,8 @@ func hasStdin() bool { return false } - isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 + isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) != 0 isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 - return isPipedFromChrDev || isPipedFromFIFO } From b3805999f357843cd138f5fab3719795fb880910 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Sat, 27 Nov 2021 17:10:27 -0600 Subject: [PATCH 146/146] Unsafe flag with base template (#1279) * unsafe flag with base template --- v2/pkg/protocols/http/build_request.go | 6 ++++-- v2/pkg/protocols/http/request.go | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 8c8a2dbc..be1d1311 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -307,10 +307,12 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte } req.Body = ioutil.NopCloser(strings.NewReader(body)) } - setHeader(req, "User-Agent", uarand.GetRandom()) + if !r.request.Unsafe { + setHeader(req, "User-Agent", uarand.GetRandom()) + } // Only set these headers on non-raw requests - if len(r.request.Raw) == 0 { + if len(r.request.Raw) == 0 && !r.request.Unsafe { setHeader(req, "Accept", "*/*") setHeader(req, "Accept-Language", "en") } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index a0bce62e..a38b56b9 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -310,9 +310,8 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if request.options.Options.Debug || request.options.Options.DebugRequests { gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL) gologger.Print().Msgf("%s", dumpedRequestString) - } + } } - var formedURL string var hostname string timeStart := time.Now()