Started refactor of template compilation + protocol building

v2/pkg/engine/engine.go

@@ -0,0 +1,9 @@
+package engine
+
+// Engine is an engine for running Nuclei Templates/Workflows.
+//
+// The engine contains multiple thread pools which allow using different
+// concurrency values per protocol executed. This was something which was
+// missing from the previous versions of nuclei.
+type Engine struct {
+}

v2/pkg/engine/engine_test.go

@@ -0,0 +1 @@
+package engine

v2/pkg/templates/compile.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"reflect"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -69,11 +70,6 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
 	options.TemplateInfo = template.Info
 	options.TemplatePath = filePath
 
-	// If no requests, and it is also not a workflow, return error.
-	if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows) == 0 {
-		return nil, fmt.Errorf("no requests defined for %s", template.ID)
-	}
-
 	// Compile the workflow request
 	if len(template.Workflows) > 0 {
 		compiled := &template.Workflow
@@ -83,56 +79,10 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
 		template.CompiledWorkflow.Options = &options
 	}
 
-	// Compile the requests found
+	if err := template.compileProtocolRequests(options); err != nil {
-	requests := []protocols.Request{}
+		return nil, err
-	if len(template.RequestsDNS) > 0 && !options.Options.OfflineHTTP {
-		for _, req := range template.RequestsDNS {
-			requests = append(requests, req)
-		}
-		template.Executer = executer.NewExecuter(requests, &options)
 	}
-	if len(template.RequestsHTTP) > 0 {
-		if options.Options.OfflineHTTP {
-			operatorsList := []*operators.Operators{}
 
-		mainLoop:
-			for _, req := range template.RequestsHTTP {
-				for _, path := range req.Path {
-					if !(strings.EqualFold(path, "{{BaseURL}}") || strings.EqualFold(path, "{{BaseURL}}/")) {
-						break mainLoop
-					}
-				}
-				operatorsList = append(operatorsList, &req.Operators)
-			}
-			if len(operatorsList) > 0 {
-				options.Operators = operatorsList
-				template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options)
-			}
-		} else {
-			for _, req := range template.RequestsHTTP {
-				requests = append(requests, req)
-			}
-			template.Executer = executer.NewExecuter(requests, &options)
-		}
-	}
-	if len(template.RequestsFile) > 0 && !options.Options.OfflineHTTP {
-		for _, req := range template.RequestsFile {
-			requests = append(requests, req)
-		}
-		template.Executer = executer.NewExecuter(requests, &options)
-	}
-	if len(template.RequestsNetwork) > 0 && !options.Options.OfflineHTTP {
-		for _, req := range template.RequestsNetwork {
-			requests = append(requests, req)
-		}
-		template.Executer = executer.NewExecuter(requests, &options)
-	}
-	if len(template.RequestsHeadless) > 0 && !options.Options.OfflineHTTP && options.Options.Headless {
-		for _, req := range template.RequestsHeadless {
-			requests = append(requests, req)
-		}
-		template.Executer = executer.NewExecuter(requests, &options)
-	}
 	if template.Executer != nil {
 		if err := template.Executer.Compile(); err != nil {
 			return nil, errors.Wrap(err, "could not compile request")
@@ -151,14 +101,95 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
 }
 
 // parseSelfContainedRequests parses the self contained template requests.
-func (t *Template) parseSelfContainedRequests() {
+func (template *Template) parseSelfContainedRequests() {
-	if !t.SelfContained {
+	if !template.SelfContained {
 		return
 	}
-	for _, request := range t.RequestsHTTP {
+	for _, request := range template.RequestsHTTP {
 		request.SelfContained = true
 	}
-	for _, request := range t.RequestsNetwork {
+	for _, request := range template.RequestsNetwork {
 		request.SelfContained = true
 	}
 }
+
+// Requests returns the total request count for the template
+func (template *Template) Requests() int {
+	return len(template.RequestsDNS) +
+		len(template.RequestsHTTP) +
+		len(template.RequestsFile) +
+		len(template.RequestsNetwork) +
+		len(template.RequestsHeadless) +
+		len(template.Workflows)
+}
+
+// compileProtocolRequests compiles all the protocol requests for the template
+func (template *Template) compileProtocolRequests(options protocols.ExecuterOptions) error {
+	templateRequests := template.Requests()
+
+	if templateRequests == 0 {
+		return fmt.Errorf("no requests defined for %s", template.ID)
+	}
+
+	if template.Options.Options.OfflineHTTP {
+		template.compileOfflineHTTPRequest(options)
+		return nil
+	}
+
+	var requests []protocols.Request
+	switch {
+	case len(template.RequestsDNS) > 0:
+		requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...)
+
+	case len(template.RequestsFile) > 0:
+		requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...)
+
+	case len(template.RequestsNetwork) > 0:
+		requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...)
+
+	case len(template.RequestsHTTP) > 0:
+		requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...)
+
+	case len(template.RequestsHeadless) > 0 && options.Options.Headless:
+		requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...)
+	}
+	template.Executer = executer.NewExecuter(requests, &options)
+	return nil
+}
+
+func (template *Template) convertRequestToProtocolsRequest(requests interface{}) []protocols.Request {
+	switch reflect.TypeOf(requests).Kind() {
+	case reflect.Slice:
+		s := reflect.ValueOf(requests)
+
+		requestSlice := make([]protocols.Request, s.Len())
+		for i := 0; i < s.Len(); i++ {
+			value := s.Index(i)
+			valueInterface := value.Interface()
+			requestSlice[i] = valueInterface.(protocols.Request)
+		}
+		return requestSlice
+	}
+	return nil
+}
+
+// compileOfflineHTTPRequest iterates all requests if offline http mode is
+// specified and collects all matchers for all the base request templates
+// (those with URL {{BaseURL}} and it's slash variation.)
+func (template *Template) compileOfflineHTTPRequest(options protocols.ExecuterOptions) {
+	operatorsList := []*operators.Operators{}
+
+mainLoop:
+	for _, req := range template.RequestsHTTP {
+		for _, path := range req.Path {
+			if !(strings.EqualFold(path, "{{BaseURL}}") || strings.EqualFold(path, "{{BaseURL}}/")) {
+				break mainLoop
+			}
+		}
+		operatorsList = append(operatorsList, &req.Operators)
+	}
+	if len(operatorsList) > 0 {
+		options.Operators = operatorsList
+		template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options)
+	}
+}