nuclei/v2/pkg/templates/cluster.go

301 lines
11 KiB
Go

package templates
import (
"fmt"
"sort"
"strings"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
cryptoutil "github.com/projectdiscovery/utils/crypto"
)
// Cluster clusters a list of templates into a lesser number if possible based
// on the similarity between the sent requests.
//
// If the attributes match, multiple requests can be clustered into a single
// request which saves time and network resources during execution.
//
// The clusterer goes through all the templates, looking for templates with a single
// HTTP/DNS/TLS request to an endpoint (multiple requests aren't clustered as of now).
//
// All the templates are iterated and any templates with request that is identical
// to the first individual request is compared for equality.
// The equality check is performed as described below -
//
// Cases where clustering is not performed (request is considered different)
// - If request contains payloads,raw,body,unsafe,req-condition,name attributes
// - If request methods,max-redirects,cookie-reuse,redirects are not equal
// - If request paths aren't identical.
// - If request headers aren't identical
// - Similarly for DNS, only identical DNS requests are clustered to a target.
// - Similarly for TLS, only identical TLS requests are clustered to a target.
//
// If multiple requests are identified as identical, they are appended to a slice.
// Finally, the engine creates a single executer with a clusteredexecuter for all templates
// in a cluster.
func Cluster(list map[string]*Template) [][]*Template {
final := [][]*Template{}
// Each protocol that can be clustered should be handled here.
for key, template := range list {
// We only cluster http and dns requests as of now.
// Take care of requests that can't be clustered first.
if len(template.RequestsHTTP) == 0 && len(template.RequestsDNS) == 0 && len(template.RequestsSSL) == 0 {
delete(list, key)
final = append(final, []*Template{template})
continue
}
delete(list, key) // delete element first so it's not found later.
var templateType types.ProtocolType
switch {
case len(template.RequestsDNS) == 1:
templateType = types.DNSProtocol
case len(template.RequestsHTTP) == 1:
templateType = types.HTTPProtocol
case len(template.RequestsSSL) == 1:
templateType = types.SSLProtocol
}
// Find any/all similar matching request that is identical to
// this one and cluster them together for http protocol only.
cluster := []*Template{}
for otherKey, other := range list {
switch templateType {
case types.DNSProtocol:
if len(other.RequestsDNS) == 0 || len(other.RequestsDNS) > 1 {
continue
} else if template.RequestsDNS[0].CanCluster(other.RequestsDNS[0]) {
delete(list, otherKey)
cluster = append(cluster, other)
}
case types.HTTPProtocol:
if len(other.RequestsHTTP) == 0 || len(other.RequestsHTTP) > 1 {
continue
} else if template.RequestsHTTP[0].CanCluster(other.RequestsHTTP[0]) {
delete(list, otherKey)
cluster = append(cluster, other)
}
case types.SSLProtocol:
if len(other.RequestsSSL) == 0 || len(other.RequestsSSL) > 1 {
continue
} else if template.RequestsSSL[0].CanCluster(other.RequestsSSL[0]) {
delete(list, otherKey)
cluster = append(cluster, other)
}
}
}
if len(cluster) > 0 {
cluster = append(cluster, template)
final = append(final, cluster)
continue
}
final = append(final, []*Template{template})
}
return final
}
// ClusterID transforms clusterization into a mathematical hash repeatable across executions with the same templates
func ClusterID(templates []*Template) string {
allIDS := make([]string, len(templates))
for tplIndex, tpl := range templates {
allIDS[tplIndex] = tpl.ID
}
sort.Strings(allIDS)
ids := strings.Join(allIDS, ",")
return cryptoutil.SHA256Sum(ids)
}
func ClusterTemplates(templatesList []*Template, options protocols.ExecutorOptions) ([]*Template, int) {
if options.Options.OfflineHTTP || options.Options.DisableClustering {
return templatesList, 0
}
templatesMap := make(map[string]*Template)
for _, v := range templatesList {
templatesMap[v.Path] = v
}
clusterCount := 0
finalTemplatesList := make([]*Template, 0, len(templatesList))
clusters := Cluster(templatesMap)
for _, cluster := range clusters {
if len(cluster) > 1 {
executerOpts := options
clusterID := fmt.Sprintf("cluster-%s", ClusterID(cluster))
for _, req := range cluster[0].RequestsDNS {
req.Options().TemplateID = clusterID
}
for _, req := range cluster[0].RequestsHTTP {
req.Options().TemplateID = clusterID
}
for _, req := range cluster[0].RequestsSSL {
req.Options().TemplateID = clusterID
}
executerOpts.TemplateID = clusterID
finalTemplatesList = append(finalTemplatesList, &Template{
ID: clusterID,
RequestsDNS: cluster[0].RequestsDNS,
RequestsHTTP: cluster[0].RequestsHTTP,
RequestsSSL: cluster[0].RequestsSSL,
Executer: NewClusterExecuter(cluster, &executerOpts),
TotalRequests: len(cluster[0].RequestsHTTP) + len(cluster[0].RequestsDNS),
})
clusterCount += len(cluster)
} else {
finalTemplatesList = append(finalTemplatesList, cluster...)
}
}
return finalTemplatesList, clusterCount
}
// ClusterExecuter executes a group of requests for a protocol for a clustered
// request. It is different from normal executers since the original
// operators are all combined and post processed after making the request.
type ClusterExecuter struct {
requests protocols.Request
operators []*clusteredOperator
templateType types.ProtocolType
options *protocols.ExecutorOptions
}
type clusteredOperator struct {
templateID string
templatePath string
templateInfo model.Info
operator *operators.Operators
}
var _ protocols.Executer = &ClusterExecuter{}
// NewClusterExecuter creates a new request executer for list of requests
func NewClusterExecuter(requests []*Template, options *protocols.ExecutorOptions) *ClusterExecuter {
executer := &ClusterExecuter{options: options}
if len(requests[0].RequestsDNS) == 1 {
executer.templateType = types.DNSProtocol
executer.requests = requests[0].RequestsDNS[0]
} else if len(requests[0].RequestsHTTP) == 1 {
executer.templateType = types.HTTPProtocol
executer.requests = requests[0].RequestsHTTP[0]
} else if len(requests[0].RequestsSSL) == 1 {
executer.templateType = types.SSLProtocol
executer.requests = requests[0].RequestsSSL[0]
}
appendOperator := func(req *Template, operator *operators.Operators) {
operator.TemplateID = req.ID
operator.ExcludeMatchers = options.ExcludeMatchers
executer.operators = append(executer.operators, &clusteredOperator{
operator: operator,
templateID: req.ID,
templateInfo: req.Info,
templatePath: req.Path,
})
}
for _, req := range requests {
if executer.templateType == types.DNSProtocol {
if req.RequestsDNS[0].CompiledOperators != nil {
appendOperator(req, req.RequestsDNS[0].CompiledOperators)
}
} else if executer.templateType == types.HTTPProtocol {
if req.RequestsHTTP[0].CompiledOperators != nil {
appendOperator(req, req.RequestsHTTP[0].CompiledOperators)
}
} else if executer.templateType == types.SSLProtocol {
if req.RequestsSSL[0].CompiledOperators != nil {
appendOperator(req, req.RequestsSSL[0].CompiledOperators)
}
}
}
return executer
}
// Compile compiles the execution generators preparing any requests possible.
func (e *ClusterExecuter) Compile() error {
return e.requests.Compile(e.options)
}
// Requests returns the total number of requests the rule will perform
func (e *ClusterExecuter) Requests() int {
var count int
count += e.requests.Requests()
return count
}
// Execute executes the protocol group and returns true or false if results were found.
func (e *ClusterExecuter) Execute(input *contextargs.Context) (bool, error) {
var results bool
inputItem := input.Clone()
if e.options.InputHelper != nil && input.MetaInput.Input != "" {
if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, e.templateType); input.MetaInput.Input == "" {
return false, nil
}
}
previous := make(map[string]interface{})
dynamicValues := make(map[string]interface{})
err := e.requests.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
for _, operator := range e.operators {
result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse)
event.InternalEvent["template-id"] = operator.templateID
event.InternalEvent["template-path"] = operator.templatePath
event.InternalEvent["template-info"] = operator.templateInfo
if result == nil && !matched {
if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil {
gologger.Warning().Msgf("Could not write failure event to output: %s\n", err)
}
continue
}
if matched && result != nil {
event.OperatorsResult = result
event.Results = e.requests.MakeResultEvent(event)
results = true
_ = writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient)
}
}
})
if err != nil && e.options.HostErrorsCache != nil {
e.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, err)
}
return results, err
}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (e *ClusterExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error {
dynamicValues := make(map[string]interface{})
inputItem := input.Clone()
if e.options.InputHelper != nil && input.MetaInput.Input != "" {
if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, e.templateType); input.MetaInput.Input == "" {
return nil
}
}
err := e.requests.ExecuteWithResults(inputItem, dynamicValues, nil, func(event *output.InternalWrappedEvent) {
for _, operator := range e.operators {
result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse)
if matched && result != nil {
event.OperatorsResult = result
event.InternalEvent["template-id"] = operator.templateID
event.InternalEvent["template-path"] = operator.templatePath
event.InternalEvent["template-info"] = operator.templateInfo
event.Results = e.requests.MakeResultEvent(event)
callback(event)
}
}
})
if err != nil && e.options.HostErrorsCache != nil {
e.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, err)
}
return err
}