Added simplehttp-only clustering impl (wip)

v2/pkg/protocols/common/clusterer/clusterer.go

@@ -0,0 +1,49 @@
+package clusterer
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+)
+
+// Cluster clusters a list of templates into a lesser number if possible based
+// on the similarity between the sent requests.
+//
+// If the attributes match, multiple requests can be clustered into a single
+// request which saves time and network resources during execution.
+func Cluster(list map[string]*templates.Template) [][]*templates.Template {
+	final := [][]*templates.Template{}
+
+	// Each protocol that can be clustered should be handled here.
+	for key, template := range list {
+		// We only cluster http requests as of now.
+		// Take care of requests that can't be clustered first.
+		if len(template.RequestsHTTP) == 0 {
+			delete(list, key)
+			final = append(final, []*templates.Template{template})
+			continue
+		}
+
+		delete(list, key) // delete element first so it's not found later.
+		// Find any/all similar matching request that is identical to
+		// this one and cluster them together for http protocol only.
+		if len(template.RequestsHTTP) == 1 {
+			cluster := []*templates.Template{}
+
+			for otherKey, other := range list {
+				if len(other.RequestsHTTP) == 0 {
+					continue
+				}
+				if template.RequestsHTTP[0].CanCluster(other.RequestsHTTP[0]) {
+					delete(list, otherKey)
+					cluster = append(cluster, other)
+				}
+			}
+			if len(cluster) > 0 {
+				cluster = append(cluster, template)
+				final = append(final, cluster)
+				continue
+			}
+		}
+		final = append(final, []*templates.Template{template})
+	}
+	return final
+}

v2/pkg/protocols/common/clusterer/clusterer_test.go

@@ -0,0 +1,77 @@
+package clusterer
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalogue"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHTTPRequestsCluster(t *testing.T) {
+	catalogue := catalogue.New("/Users/ice3man/nuclei-templates")
+	templatesList, err := catalogue.GetTemplatePath("/Users/ice3man/nuclei-templates")
+	require.Nil(t, err, "could not get templates")
+
+	protocolinit.Init(&types.Options{})
+	list := make(map[string]*templates.Template)
+	for _, template := range templatesList {
+		executerOpts := &protocols.ExecuterOptions{
+			Output:      &mockOutput{},
+			Options:     &types.Options{},
+			Progress:    nil,
+			Catalogue:   catalogue,
+			RateLimiter: nil,
+			ProjectFile: nil,
+		}
+		t, err := templates.Parse(template, executerOpts)
+		if err != nil {
+			continue
+		}
+		if _, ok := list[t.ID]; !ok {
+			list[t.ID] = t
+		} else {
+			//	log.Fatalf("Duplicate template found: %v\n", t)
+		}
+	}
+
+	totalClusterCount := 0
+	totalRequestsSentNew := 0
+	new := Cluster(list)
+	for i, cluster := range new {
+		if len(cluster) == 1 {
+			continue
+		}
+		fmt.Printf("[%d] cluster created:\n", i)
+		for _, request := range cluster {
+			totalClusterCount++
+			fmt.Printf("\t%v\n", request.ID)
+		}
+		totalRequestsSentNew++
+	}
+	fmt.Printf("Reduced %d requests to %d via clustering\n", totalClusterCount, totalRequestsSentNew)
+}
+
+type mockOutput struct{}
+
+// Close closes the output writer interface
+func (m *mockOutput) Close() {}
+
+// Colorizer returns the colorizer instance for writer
+func (m *mockOutput) Colorizer() aurora.Aurora {
+	return nil
+}
+
+// Write writes the event to file and/or screen.
+func (m *mockOutput) Write(*output.ResultEvent) error {
+	return nil
+}
+
+// Request writes a log the requests trace log
+func (m *mockOutput) Request(templateID, url, requestType string, err error) {}

v2/pkg/protocols/common/compare/compare.go

@@ -0,0 +1,37 @@
+package compare
+
+import "strings"
+
+// StringSlice compares two string slices for equality
+func StringSlice(a, b []string) bool {
+	// If one is nil, the other must also be nil.
+	if (a == nil) != (b == nil) {
+		return false
+	}
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if !strings.EqualFold(a[i], b[i]) {
+			return false
+		}
+	}
+	return true
+}
+
+// StringMap compares two string maps for equality
+func StringMap(a, b map[string]string) bool {
+	// If one is nil, the other must also be nil.
+	if (a == nil) != (b == nil) {
+		return false
+	}
+	if len(a) != len(b) {
+		return false
+	}
+	for k, v := range a {
+		if w, ok := b[k]; !ok || !strings.EqualFold(v, w) {
+			return false
+		}
+	}
+	return true
+}

v2/pkg/protocols/common/executer/executer.go

@@ -18,6 +18,11 @@ func NewExecuter(requests []protocols.Request, options *protocols.ExecuterOption
 	return &Executer{requests: requests, options: options}
 }
 
+// GetRequests returns the requests the rule will perform
+func (e *Executer) GetRequests() []protocols.Request {
+	return e.requests
+}
+
 // Compile compiles the execution generators preparing any requests possible.
 func (e *Executer) Compile() error {
 	for _, request := range e.requests {

v2/pkg/protocols/http/cluster.go

@@ -0,0 +1,29 @@
+package http
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/compare"
+)
+
+// CanCluster returns true if the request can be clustered.
+//
+// This used by the clustering engine to decide whether two requests
+// are similar enough to be considered one and can be checked by
+// just adding the matcher/extractors for the request and the correct IDs.
+func (r *Request) CanCluster(other *Request) bool {
+	if len(r.Payloads) > 0 || len(r.Raw) > 0 || len(r.Body) > 0 || r.Unsafe {
+		return false
+	}
+	if r.Method != other.Method ||
+		r.MaxRedirects != other.MaxRedirects ||
+		r.CookieReuse != other.CookieReuse ||
+		r.Redirects != other.Redirects {
+		return false
+	}
+	if !compare.StringSlice(r.Path, other.Path) {
+		return false
+	}
+	if !compare.StringMap(r.Headers, other.Headers) {
+		return false
+	}
+	return true
+}