Merge pull request #1123 from EndPositive/remote-template-workflow-lists

Remote template workflow lists

integration_tests/loader/basic.yaml

@@ -0,0 +1,10 @@
+id: workflow-example
+
+info:
+  name: Test Workflow Template
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-1.yaml
+  - template: workflow/match-2.yaml

integration_tests/loader/condition-matched.yaml

@@ -0,0 +1,11 @@
+id: condition-matched-workflow
+
+info:
+  name: Condition Matched Workflow
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-1.yaml
+    subtemplates:
+      - template: workflow/match-2.yaml

integration_tests/loader/get-headers.yaml

@@ -0,0 +1,17 @@
+id: basic-get-headers
+
+info:
+  name: Basic GET Headers Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    headers:
+      test: nuclei
+    matchers:
+      - type: word
+        words:
+          - "This is test headers matcher text"

integration_tests/loader/get.yaml

@@ -0,0 +1,15 @@
+id: basic-get
+
+info:
+  name: Basic GET Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "This is test matcher text"

integration_tests/loader/template-list.yaml

@@ -0,0 +1,2 @@
+loader/get.yaml
+loader/get-headers.yaml

integration_tests/loader/workflow-list.yaml

@@ -0,0 +1,2 @@
+loader/basic.yaml
+loader/condition-matched.yaml

v2/cmd/integration-test/integration-test.go

@@ -26,6 +26,7 @@ func main() {
 		"network":  networkTestcases,
 		"dns":      dnsTestCases,
 		"workflow": workflowTestcases,
+		"loader":   loaderTestcases,
 	}
 	for proto, tests := range protocolTests {
 		if protocol == "" || protocol == proto {

v2/cmd/integration-test/loader.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"fmt"
+	"github.com/julienschmidt/httprouter"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+)
+
+var loaderTestcases = map[string]testutils.TestCase{
+	"loader/template-list.yaml":             &remoteTemplateList{},
+	"loader/workflow-list.yaml":             &remoteWorkflowList{},
+	"loader/nonexistent-template-list.yaml": &nonExistentTemplateList{},
+	"loader/nonexistent-workflow-list.yaml": &nonExistentWorkflowList{},
+}
+
+type remoteTemplateList struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *remoteTemplateList) Execute(templateList string) error {
+	router := httprouter.New()
+
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "This is test matcher text")
+		if strings.EqualFold(r.Header.Get("test"), "nuclei") {
+			fmt.Fprintf(w, "This is test headers matcher text")
+		}
+	}))
+
+	router.GET("/template_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		file, err := os.ReadFile(templateList)
+		if err != nil {
+			w.WriteHeader(500)
+		}
+		_, err = w.Write(file)
+		if err != nil {
+			w.WriteHeader(500)
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list")
+	if err != nil {
+		return err
+	}
+	if len(results) != 2 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type remoteWorkflowList struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *remoteWorkflowList) Execute(workflowList string) error {
+	router := httprouter.New()
+
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "This is test matcher text")
+		if strings.EqualFold(r.Header.Get("test"), "nuclei") {
+			fmt.Fprintf(w, "This is test headers matcher text")
+		}
+	}))
+
+	router.GET("/workflow_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		file, err := os.ReadFile(workflowList)
+		if err != nil {
+			w.WriteHeader(500)
+		}
+		_, err = w.Write(file)
+		if err != nil {
+			w.WriteHeader(500)
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list")
+	if err != nil {
+		return err
+	}
+	if len(results) != 3 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type nonExistentTemplateList struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error {
+	router := httprouter.New()
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/404")
+	if err == nil {
+		return fmt.Errorf("expected error for nonexisting workflow url")
+	}
+
+	return nil
+}
+
+type nonExistentWorkflowList struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error {
+	router := httprouter.New()
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/404")
+	if err == nil {
+		return fmt.Errorf("expected error for nonexisting workflow url")
+	}
+
+	return nil
+}

v2/cmd/nuclei/main.go

@@ -54,8 +54,10 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	createGroup(flagSet, "templates", "Templates",
 		flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"),
+		flagSet.StringSliceVarP(&options.TemplateURLs, "template-urls", "tu", []string{}, "URL to a list of templates"),
 		flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"),
 		flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "workflow or workflow directory paths to include in the scan"),
+		flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-urls", "wu", []string{}, "URL to a list of workflows to run"),
 		flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
 		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
 	)

v2/internal/runner/runner.go

@@ -366,7 +366,9 @@ func (r *Runner) RunEnumeration() error {
 
 	loaderConfig := loader.Config{
 		Templates:          r.options.Templates,
+		TemplateURLs:       r.options.TemplateURLs,
 		Workflows:          r.options.Workflows,
+		WorkflowURLs:       r.options.WorkflowURLs,
 		ExcludeTemplates:   r.options.ExcludedTemplates,
 		Tags:               r.options.Tags,
 		ExcludeTags:        r.options.ExcludeTags,

v2/internal/testutils/integration.go

@@ -12,15 +12,15 @@ import (
 
 // RunNucleiTemplateAndGetResults returns a list of results for a template
 func RunNucleiTemplateAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
-	return runNucleiAndGetResults(true, template, url, debug, extra...)
+	return RunNucleiAndGetResults(true, template, url, debug, extra...)
 }
 
 // RunNucleiWorkflowAndGetResults returns a list of results for a workflow
 func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
-	return runNucleiAndGetResults(false, template, url, debug, extra...)
+	return RunNucleiAndGetResults(false, template, url, debug, extra...)
 }
 
-func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) {
+func RunNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) {
 	var templateOrWorkflowFlag string
 	if isTemplate {
 		templateOrWorkflowFlag = "-t"
@@ -28,11 +28,22 @@ func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, e
 		templateOrWorkflowFlag = "-w"
 	}
 
-	cmd := exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-silent")
+	return RunNucleiBareArgsAndGetResults(debug, append([]string{
+		templateOrWorkflowFlag,
+		template,
+		"-target",
+		url,
+	}, extra...)...)
+}
+
+func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, error) {
+	cmd := exec.Command("./nuclei")
 	if debug {
-		cmd = exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-debug")
+		cmd.Args = append(cmd.Args, "-debug")
 		cmd.Stderr = os.Stderr
 		fmt.Println(cmd.String())
+	} else {
+		cmd.Args = append(cmd.Args, "-silent")
 	}
 	cmd.Args = append(cmd.Args, extra...)
 	data, err := cmd.Output()

v2/pkg/catalog/loader/loader.go

@@ -15,7 +15,9 @@ import (
 // Config contains the configuration options for the loader
 type Config struct {
 	Templates        []string
+	TemplateURLs     []string
 	Workflows        []string
+	WorkflowURLs     []string
 	ExcludeTemplates []string
 	IncludeTemplates []string
 
@@ -37,6 +39,7 @@ type Store struct {
 	pathFilter     *filter.PathFilter
 	config         *Config
 	finalTemplates []string
+	finalWorkflows []string
 
 	templates []*templates.Template
 	workflows []*templates.Template
@@ -61,13 +64,24 @@ func New(config *Config) (*Store, error) {
 			IncludedTemplates: config.IncludeTemplates,
 			ExcludedTemplates: config.ExcludeTemplates,
 		}, config.Catalog),
+		finalTemplates: config.Templates,
+		finalWorkflows: config.Workflows,
+	}
+
+	if len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 {
+		remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs)
+		if err != nil {
+			return store, err
+		}
+		store.finalTemplates = append(store.finalTemplates, remoteTemplates...)
+		store.finalWorkflows = append(store.finalWorkflows, remoteWorkflows...)
 	}
 
 	// Handle a case with no templates or workflows, where we use base directory
-	if len(config.Templates) == 0 && len(config.Workflows) == 0 {
+	if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 {
-		config.Templates = append(config.Templates, config.TemplatesDirectory)
+		store.finalTemplates = []string{config.TemplatesDirectory}
 	}
-	store.finalTemplates = append(store.finalTemplates, config.Templates...)
+
 	return store, nil
 }
 
@@ -90,7 +104,7 @@ func (store *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
 // the complete compiled templates for a nuclei execution configuration.
 func (store *Store) Load() {
 	store.templates = store.LoadTemplates(store.finalTemplates)
-	store.workflows = store.LoadWorkflows(store.config.Workflows)
+	store.workflows = store.LoadWorkflows(store.finalWorkflows)
 }
 
 // ValidateTemplates takes a list of templates and validates them

v2/pkg/catalog/loader/remote_loader.go

@@ -0,0 +1,94 @@
+package loader
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/pkg/errors"
+	"net/http"
+	"strings"
+)
+
+type ContentType string
+
+const (
+	Template ContentType = "Template"
+	Workflow ContentType = "Workflow"
+)
+
+type RemoteContentError struct {
+	Content []string
+	Type    ContentType
+	Error   error
+}
+
+func getRemoteTemplatesAndWorkflows(templateURLs []string, workflowURLs []string) ([]string, []string, error) {
+	remoteContentErrorChannel := make(chan RemoteContentError)
+
+	for _, templateURL := range templateURLs {
+		go getRemoteContent(templateURL, remoteContentErrorChannel, Template)
+	}
+	for _, workflowURL := range workflowURLs {
+		go getRemoteContent(workflowURL, remoteContentErrorChannel, Workflow)
+	}
+
+	var remoteTemplateList []string
+	var remoteWorkFlowList []string
+	var err error
+	for i := 0; i < (len(templateURLs) + len(workflowURLs)); i++ {
+		remoteContentError := <-remoteContentErrorChannel
+		if remoteContentError.Error != nil {
+			if err != nil {
+				err = errors.New(remoteContentError.Error.Error() + ": " + err.Error())
+			} else {
+				err = remoteContentError.Error
+			}
+		} else {
+			if remoteContentError.Type == Template {
+				remoteTemplateList = append(remoteTemplateList, remoteContentError.Content...)
+			} else if remoteContentError.Type == Workflow {
+				remoteWorkFlowList = append(remoteWorkFlowList, remoteContentError.Content...)
+			}
+		}
+	}
+
+	return remoteTemplateList, remoteWorkFlowList, err
+}
+
+func getRemoteContent(URL string, w chan<- RemoteContentError, contentType ContentType) {
+	response, err := http.Get(URL)
+	if err != nil {
+		w <- RemoteContentError{
+			Error: err,
+		}
+		return
+	}
+	defer response.Body.Close()
+	if response.StatusCode < 200 || response.StatusCode > 299 {
+		w <- RemoteContentError{
+			Error: fmt.Errorf("get \"%s\": unexpect status %d", URL, response.StatusCode),
+		}
+		return
+	}
+
+	scanner := bufio.NewScanner(response.Body)
+	var templateList []string
+	for scanner.Scan() {
+		text := strings.TrimSpace(scanner.Text())
+		if text == "" {
+			continue
+		}
+		templateList = append(templateList, text)
+	}
+
+	if err := scanner.Err(); err != nil {
+		w <- RemoteContentError{
+			Error: errors.Wrap(err, "get \"%s\""),
+		}
+		return
+	}
+
+	w <- RemoteContentError{
+		Content: templateList,
+		Type:    contentType,
+	}
+}

v2/pkg/types/types.go

@@ -15,8 +15,12 @@ type Options struct {
 	ExcludeTags goflags.NormalizedStringSlice
 	// Workflows specifies any workflows to run by nuclei
 	Workflows goflags.StringSlice
+	// WorkflowURLs specifies URLs to a list of workflows to use
+	WorkflowURLs goflags.StringSlice
 	// Templates specifies the template/templates to use
 	Templates goflags.StringSlice
+	// TemplateURLs specifies URLs to a list of templates to use
+	TemplateURLs goflags.StringSlice
 	// 	ExcludedTemplates  specifies the template/templates to exclude
 	ExcludedTemplates goflags.StringSlice
 	// CustomHeaders is the list of custom global headers to send with each request.