diff --git a/v2/pkg/protocols/common/clusterer/clusterer.go b/v2/pkg/protocols/common/clusterer/clusterer.go new file mode 100644 index 00000000..29c1a309 --- /dev/null +++ b/v2/pkg/protocols/common/clusterer/clusterer.go @@ -0,0 +1,49 @@ +package clusterer + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/templates" +) + +// Cluster clusters a list of templates into a lesser number if possible based +// on the similarity between the sent requests. +// +// If the attributes match, multiple requests can be clustered into a single +// request which saves time and network resources during execution. +func Cluster(list map[string]*templates.Template) [][]*templates.Template { + final := [][]*templates.Template{} + + // Each protocol that can be clustered should be handled here. + for key, template := range list { + // We only cluster http requests as of now. + // Take care of requests that can't be clustered first. + if len(template.RequestsHTTP) == 0 { + delete(list, key) + final = append(final, []*templates.Template{template}) + continue + } + + delete(list, key) // delete element first so it's not found later. + // Find any/all similar matching request that is identical to + // this one and cluster them together for http protocol only. + if len(template.RequestsHTTP) == 1 { + cluster := []*templates.Template{} + + for otherKey, other := range list { + if len(other.RequestsHTTP) == 0 { + continue + } + if template.RequestsHTTP[0].CanCluster(other.RequestsHTTP[0]) { + delete(list, otherKey) + cluster = append(cluster, other) + } + } + if len(cluster) > 0 { + cluster = append(cluster, template) + final = append(final, cluster) + continue + } + } + final = append(final, []*templates.Template{template}) + } + return final +} diff --git a/v2/pkg/protocols/common/clusterer/clusterer_test.go b/v2/pkg/protocols/common/clusterer/clusterer_test.go new file mode 100644 index 00000000..1622b12f --- /dev/null +++ b/v2/pkg/protocols/common/clusterer/clusterer_test.go @@ -0,0 +1,77 @@ +package clusterer + +import ( + "fmt" + "testing" + + "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/nuclei/v2/pkg/catalogue" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/stretchr/testify/require" +) + +func TestHTTPRequestsCluster(t *testing.T) { + catalogue := catalogue.New("/Users/ice3man/nuclei-templates") + templatesList, err := catalogue.GetTemplatePath("/Users/ice3man/nuclei-templates") + require.Nil(t, err, "could not get templates") + + protocolinit.Init(&types.Options{}) + list := make(map[string]*templates.Template) + for _, template := range templatesList { + executerOpts := &protocols.ExecuterOptions{ + Output: &mockOutput{}, + Options: &types.Options{}, + Progress: nil, + Catalogue: catalogue, + RateLimiter: nil, + ProjectFile: nil, + } + t, err := templates.Parse(template, executerOpts) + if err != nil { + continue + } + if _, ok := list[t.ID]; !ok { + list[t.ID] = t + } else { + // log.Fatalf("Duplicate template found: %v\n", t) + } + } + + totalClusterCount := 0 + totalRequestsSentNew := 0 + new := Cluster(list) + for i, cluster := range new { + if len(cluster) == 1 { + continue + } + fmt.Printf("[%d] cluster created:\n", i) + for _, request := range cluster { + totalClusterCount++ + fmt.Printf("\t%v\n", request.ID) + } + totalRequestsSentNew++ + } + fmt.Printf("Reduced %d requests to %d via clustering\n", totalClusterCount, totalRequestsSentNew) +} + +type mockOutput struct{} + +// Close closes the output writer interface +func (m *mockOutput) Close() {} + +// Colorizer returns the colorizer instance for writer +func (m *mockOutput) Colorizer() aurora.Aurora { + return nil +} + +// Write writes the event to file and/or screen. +func (m *mockOutput) Write(*output.ResultEvent) error { + return nil +} + +// Request writes a log the requests trace log +func (m *mockOutput) Request(templateID, url, requestType string, err error) {} diff --git a/v2/pkg/protocols/common/compare/compare.go b/v2/pkg/protocols/common/compare/compare.go new file mode 100644 index 00000000..683b5fac --- /dev/null +++ b/v2/pkg/protocols/common/compare/compare.go @@ -0,0 +1,37 @@ +package compare + +import "strings" + +// StringSlice compares two string slices for equality +func StringSlice(a, b []string) bool { + // If one is nil, the other must also be nil. + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for i := range a { + if !strings.EqualFold(a[i], b[i]) { + return false + } + } + return true +} + +// StringMap compares two string maps for equality +func StringMap(a, b map[string]string) bool { + // If one is nil, the other must also be nil. + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for k, v := range a { + if w, ok := b[k]; !ok || !strings.EqualFold(v, w) { + return false + } + } + return true +} diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 12f8a365..a88c077f 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -18,6 +18,11 @@ func NewExecuter(requests []protocols.Request, options *protocols.ExecuterOption return &Executer{requests: requests, options: options} } +// GetRequests returns the requests the rule will perform +func (e *Executer) GetRequests() []protocols.Request { + return e.requests +} + // Compile compiles the execution generators preparing any requests possible. func (e *Executer) Compile() error { for _, request := range e.requests { diff --git a/v2/pkg/protocols/http/cluster.go b/v2/pkg/protocols/http/cluster.go new file mode 100644 index 00000000..68107c9e --- /dev/null +++ b/v2/pkg/protocols/http/cluster.go @@ -0,0 +1,29 @@ +package http + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/compare" +) + +// CanCluster returns true if the request can be clustered. +// +// This used by the clustering engine to decide whether two requests +// are similar enough to be considered one and can be checked by +// just adding the matcher/extractors for the request and the correct IDs. +func (r *Request) CanCluster(other *Request) bool { + if len(r.Payloads) > 0 || len(r.Raw) > 0 || len(r.Body) > 0 || r.Unsafe { + return false + } + if r.Method != other.Method || + r.MaxRedirects != other.MaxRedirects || + r.CookieReuse != other.CookieReuse || + r.Redirects != other.Redirects { + return false + } + if !compare.StringSlice(r.Path, other.Path) { + return false + } + if !compare.StringMap(r.Headers, other.Headers) { + return false + } + return true +}