diff --git a/v2/pkg/protocols/dns/cluster.go b/v2/pkg/protocols/dns/cluster.go index 560cf319..f2f57347 100644 --- a/v2/pkg/protocols/dns/cluster.go +++ b/v2/pkg/protocols/dns/cluster.go @@ -12,9 +12,16 @@ func (request *Request) CanCluster(other *Request) bool { if request.Name != other.Name || request.class != other.class || request.Retries != other.Retries || - request.question != other.question || - *request.Recursion != *other.Recursion { + request.question != other.question { return false } + if request.Recursion != nil { + if other.Recursion == nil { + return false + } + if *request.Recursion != *other.Recursion { + return false + } + } return true } diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index aba6be81..3910714f 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -13,6 +13,7 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" @@ -77,6 +78,17 @@ type Request struct { options *protocols.ExecuterOptions } +// CanCluster returns true if the request can be clustered. +func (request *Request) CanCluster(other *Request) bool { + if len(request.CiperSuites) > 0 || request.MinVersion != "" || request.MaxVersion != "" { + return false + } + if request.Address != other.Address || request.ScanMode != other.ScanMode { + return false + } + return true +} + // Compile compiles the request generators preparing any requests possible. func (request *Request) Compile(options *protocols.ExecuterOptions) error { request.options = options @@ -126,6 +138,11 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { return nil } +// Options returns executer options for http request +func (r *Request) Options() *protocols.ExecuterOptions { + return r.options +} + // Requests returns the total number of requests the rule will perform func (request *Request) Requests() int { return 1 @@ -315,9 +332,9 @@ func (request *Request) Type() templateTypes.ProtocolType { func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ - TemplateID: types.ToString(request.options.TemplateID), - TemplatePath: types.ToString(request.options.TemplatePath), - Info: request.options.TemplateInfo, + TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), + Info: wrapped.InternalEvent["template-info"].(model.Info), Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["host"]), diff --git a/v2/pkg/templates/cluster.go b/v2/pkg/templates/cluster.go index 0125e933..b7d7be70 100644 --- a/v2/pkg/templates/cluster.go +++ b/v2/pkg/templates/cluster.go @@ -23,7 +23,7 @@ import ( // request which saves time and network resources during execution. // // The clusterer goes through all the templates, looking for templates with a single -// HTTP/DNS request to an endpoint (multiple requests aren't clustered as of now). +// HTTP/DNS/TLS request to an endpoint (multiple requests aren't clustered as of now). // // All the templates are iterated and any templates with request that is identical // to the first individual request is compared for equality. @@ -35,6 +35,7 @@ import ( // - If request paths aren't identical. // - If request headers aren't identical // - Similarly for DNS, only identical DNS requests are clustered to a target. +// - Similarly for TLS, only identical TLS requests are clustered to a target. // // If multiple requests are identified as identical, they are appended to a slice. // Finally, the engine creates a single executer with a clusteredexecuter for all templates @@ -46,33 +47,46 @@ func Cluster(list map[string]*Template) [][]*Template { for key, template := range list { // We only cluster http and dns requests as of now. // Take care of requests that can't be clustered first. - if len(template.RequestsHTTP) == 0 && len(template.RequestsDNS) == 0 { + if len(template.RequestsHTTP) == 0 && len(template.RequestsDNS) == 0 && len(template.RequestsSSL) == 0 { delete(list, key) final = append(final, []*Template{template}) continue } delete(list, key) // delete element first so it's not found later. + var templateType types.ProtocolType + switch { + case len(template.RequestsDNS) == 1: + templateType = types.DNSProtocol + case len(template.RequestsHTTP) == 1: + templateType = types.HTTPProtocol + case len(template.RequestsSSL) == 1: + templateType = types.SSLProtocol + } + // Find any/all similar matching request that is identical to // this one and cluster them together for http protocol only. cluster := []*Template{} - if len(template.RequestsDNS) == 1 { - for otherKey, other := range list { + for otherKey, other := range list { + switch templateType { + case types.DNSProtocol: if len(other.RequestsDNS) == 0 || len(other.RequestsDNS) > 1 { continue - } - if template.RequestsDNS[0].CanCluster(other.RequestsDNS[0]) { + } else if template.RequestsDNS[0].CanCluster(other.RequestsDNS[0]) { delete(list, otherKey) cluster = append(cluster, other) } - } - } - if len(template.RequestsHTTP) == 1 { - for otherKey, other := range list { + case types.HTTPProtocol: if len(other.RequestsHTTP) == 0 || len(other.RequestsHTTP) > 1 { continue + } else if template.RequestsHTTP[0].CanCluster(other.RequestsHTTP[0]) { + delete(list, otherKey) + cluster = append(cluster, other) } - if template.RequestsHTTP[0].CanCluster(other.RequestsHTTP[0]) { + case types.SSLProtocol: + if len(other.RequestsSSL) == 0 || len(other.RequestsSSL) > 1 { + continue + } else if template.RequestsSSL[0].CanCluster(other.RequestsSSL[0]) { delete(list, otherKey) cluster = append(cluster, other) } @@ -123,11 +137,15 @@ func ClusterTemplates(templatesList []*Template, options protocols.ExecuterOptio for _, req := range cluster[0].RequestsHTTP { req.Options().TemplateID = clusterID } + for _, req := range cluster[0].RequestsSSL { + req.Options().TemplateID = clusterID + } executerOpts.TemplateID = clusterID finalTemplatesList = append(finalTemplatesList, &Template{ ID: clusterID, RequestsDNS: cluster[0].RequestsDNS, RequestsHTTP: cluster[0].RequestsHTTP, + RequestsSSL: cluster[0].RequestsSSL, Executer: NewClusterExecuter(cluster, &executerOpts), TotalRequests: len(cluster[0].RequestsHTTP) + len(cluster[0].RequestsDNS), }) @@ -167,6 +185,9 @@ func NewClusterExecuter(requests []*Template, options *protocols.ExecuterOptions } else if len(requests[0].RequestsHTTP) == 1 { executer.templateType = types.HTTPProtocol executer.requests = requests[0].RequestsHTTP[0] + } else if len(requests[0].RequestsSSL) == 1 { + executer.templateType = types.SSLProtocol + executer.requests = requests[0].RequestsSSL[0] } appendOperator := func(req *Template, operator *operators.Operators) { operator.TemplateID = req.ID @@ -188,6 +209,10 @@ func NewClusterExecuter(requests []*Template, options *protocols.ExecuterOptions if req.RequestsHTTP[0].CompiledOperators != nil { appendOperator(req, req.RequestsHTTP[0].CompiledOperators) } + } else if executer.templateType == types.SSLProtocol { + if req.RequestsSSL[0].CompiledOperators != nil { + appendOperator(req, req.RequestsSSL[0].CompiledOperators) + } } } return executer diff --git a/v2/pkg/templates/cluster_test.go b/v2/pkg/templates/cluster_test.go index dac8432f..31e54a6a 100644 --- a/v2/pkg/templates/cluster_test.go +++ b/v2/pkg/templates/cluster_test.go @@ -1 +1,58 @@ package templates + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" + "github.com/stretchr/testify/require" +) + +func TestClusterTemplates(t *testing.T) { + tests := []struct { + name string + templates map[string]*Template + expected [][]*Template + }{ + { + name: "http-cluster-get", + templates: map[string]*Template{ + "first.yaml": {RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}}, + "second.yaml": {RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}}, + }, + expected: [][]*Template{{ + {RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}}, + {RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}}, + }}, + }, + { + name: "no-http-cluster", + templates: map[string]*Template{ + "first.yaml": {RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}/random"}}}}, + "second.yaml": {RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}/another"}}}}, + }, + expected: [][]*Template{ + {{RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}/random"}}}}}, + {{RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}/another"}}}}}, + }, + }, + { + name: "dns-cluster", + templates: map[string]*Template{ + "first.yaml": {RequestsDNS: []*dns.Request{{Name: "{{Hostname}}"}}}, + "second.yaml": {RequestsDNS: []*dns.Request{{Name: "{{Hostname}}"}}}, + }, + expected: [][]*Template{{ + {RequestsDNS: []*dns.Request{{Name: "{{Hostname}}"}}}, + {RequestsDNS: []*dns.Request{{Name: "{{Hostname}}"}}}, + }}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + returned := Cluster(test.templates) + require.ElementsMatch(t, returned, test.expected, "could not get cluster results") + }) + } +}