diff --git a/integration_tests/loader/basic.yaml b/integration_tests/loader/basic.yaml new file mode 100644 index 00000000..f49193b1 --- /dev/null +++ b/integration_tests/loader/basic.yaml @@ -0,0 +1,10 @@ +id: workflow-example + +info: + name: Test Workflow Template + author: pdteam + severity: info + +workflows: + - template: workflow/match-1.yaml + - template: workflow/match-2.yaml \ No newline at end of file diff --git a/integration_tests/loader/condition-matched.yaml b/integration_tests/loader/condition-matched.yaml new file mode 100644 index 00000000..8b0a6573 --- /dev/null +++ b/integration_tests/loader/condition-matched.yaml @@ -0,0 +1,11 @@ +id: condition-matched-workflow + +info: + name: Condition Matched Workflow + author: pdteam + severity: info + +workflows: + - template: workflow/match-1.yaml + subtemplates: + - template: workflow/match-2.yaml \ No newline at end of file diff --git a/integration_tests/loader/get-headers.yaml b/integration_tests/loader/get-headers.yaml new file mode 100644 index 00000000..bae36705 --- /dev/null +++ b/integration_tests/loader/get-headers.yaml @@ -0,0 +1,17 @@ +id: basic-get-headers + +info: + name: Basic GET Headers Request + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + headers: + test: nuclei + matchers: + - type: word + words: + - "This is test headers matcher text" \ No newline at end of file diff --git a/integration_tests/loader/get.yaml b/integration_tests/loader/get.yaml new file mode 100644 index 00000000..c7e07e8c --- /dev/null +++ b/integration_tests/loader/get.yaml @@ -0,0 +1,15 @@ +id: basic-get + +info: + name: Basic GET Request + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + matchers: + - type: word + words: + - "This is test matcher text" \ No newline at end of file diff --git a/integration_tests/loader/template-list.yaml b/integration_tests/loader/template-list.yaml new file mode 100644 index 00000000..fae00d6c --- /dev/null +++ b/integration_tests/loader/template-list.yaml @@ -0,0 +1,2 @@ +loader/get.yaml +loader/get-headers.yaml diff --git a/integration_tests/loader/workflow-list.yaml b/integration_tests/loader/workflow-list.yaml new file mode 100644 index 00000000..3f56730e --- /dev/null +++ b/integration_tests/loader/workflow-list.yaml @@ -0,0 +1,2 @@ +loader/basic.yaml +loader/condition-matched.yaml diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index baa743e8..94e6c215 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -26,6 +26,7 @@ func main() { "network": networkTestcases, "dns": dnsTestCases, "workflow": workflowTestcases, + "loader": loaderTestcases, } for proto, tests := range protocolTests { if protocol == "" || protocol == proto { diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go new file mode 100644 index 00000000..8c44bc20 --- /dev/null +++ b/v2/cmd/integration-test/loader.go @@ -0,0 +1,122 @@ +package main + +import ( + "fmt" + "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "net/http" + "net/http/httptest" + "os" + "strings" +) + +var loaderTestcases = map[string]testutils.TestCase{ + "loader/template-list.yaml": &remoteTemplateList{}, + "loader/workflow-list.yaml": &remoteWorkflowList{}, + "loader/nonexistent-template-list.yaml": &nonExistentTemplateList{}, + "loader/nonexistent-workflow-list.yaml": &nonExistentWorkflowList{}, +} + +type remoteTemplateList struct{} + +// Execute executes a test case and returns an error if occurred +func (h *remoteTemplateList) Execute(templateList string) error { + router := httprouter.New() + + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "This is test matcher text") + if strings.EqualFold(r.Header.Get("test"), "nuclei") { + fmt.Fprintf(w, "This is test headers matcher text") + } + })) + + router.GET("/template_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + file, err := os.ReadFile(templateList) + if err != nil { + w.WriteHeader(500) + } + _, err = w.Write(file) + if err != nil { + w.WriteHeader(500) + } + })) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list") + if err != nil { + return err + } + if len(results) != 2 { + return errIncorrectResultsCount(results) + } + return nil +} + +type remoteWorkflowList struct{} + +// Execute executes a test case and returns an error if occurred +func (h *remoteWorkflowList) Execute(workflowList string) error { + router := httprouter.New() + + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "This is test matcher text") + if strings.EqualFold(r.Header.Get("test"), "nuclei") { + fmt.Fprintf(w, "This is test headers matcher text") + } + })) + + router.GET("/workflow_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + file, err := os.ReadFile(workflowList) + if err != nil { + w.WriteHeader(500) + } + _, err = w.Write(file) + if err != nil { + w.WriteHeader(500) + } + })) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list") + if err != nil { + return err + } + if len(results) != 3 { + return errIncorrectResultsCount(results) + } + return nil +} + +type nonExistentTemplateList struct{} + +// Execute executes a test case and returns an error if occurred +func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error { + router := httprouter.New() + ts := httptest.NewServer(router) + defer ts.Close() + + _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/404") + if err == nil { + return fmt.Errorf("expected error for nonexisting workflow url") + } + + return nil +} + +type nonExistentWorkflowList struct{} + +// Execute executes a test case and returns an error if occurred +func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error { + router := httprouter.New() + ts := httptest.NewServer(router) + defer ts.Close() + + _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/404") + if err == nil { + return fmt.Errorf("expected error for nonexisting workflow url") + } + + return nil +} diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index f68e0020..08905537 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -54,8 +54,10 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "templates", "Templates", flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"), + flagSet.StringSliceVarP(&options.TemplateURLs, "template-urls", "tu", []string{}, "URL to a list of templates"), flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"), flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "workflow or workflow directory paths to include in the scan"), + flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-urls", "wu", []string{}, "URL to a list of workflows to run"), flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"), flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index ca74f6a4..de1caff9 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -366,7 +366,9 @@ func (r *Runner) RunEnumeration() error { loaderConfig := loader.Config{ Templates: r.options.Templates, + TemplateURLs: r.options.TemplateURLs, Workflows: r.options.Workflows, + WorkflowURLs: r.options.WorkflowURLs, ExcludeTemplates: r.options.ExcludedTemplates, Tags: r.options.Tags, ExcludeTags: r.options.ExcludeTags, diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index 8c5491d2..c1cdfcc8 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -12,15 +12,15 @@ import ( // RunNucleiTemplateAndGetResults returns a list of results for a template func RunNucleiTemplateAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { - return runNucleiAndGetResults(true, template, url, debug, extra...) + return RunNucleiAndGetResults(true, template, url, debug, extra...) } // RunNucleiWorkflowAndGetResults returns a list of results for a workflow func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { - return runNucleiAndGetResults(false, template, url, debug, extra...) + return RunNucleiAndGetResults(false, template, url, debug, extra...) } -func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) { +func RunNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) { var templateOrWorkflowFlag string if isTemplate { templateOrWorkflowFlag = "-t" @@ -28,11 +28,22 @@ func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, e templateOrWorkflowFlag = "-w" } - cmd := exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-silent") + return RunNucleiBareArgsAndGetResults(debug, append([]string{ + templateOrWorkflowFlag, + template, + "-target", + url, + }, extra...)...) +} + +func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, error) { + cmd := exec.Command("./nuclei") if debug { - cmd = exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-debug") + cmd.Args = append(cmd.Args, "-debug") cmd.Stderr = os.Stderr fmt.Println(cmd.String()) + } else { + cmd.Args = append(cmd.Args, "-silent") } cmd.Args = append(cmd.Args, extra...) data, err := cmd.Output() diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index eac22b4a..6d914ae5 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -15,7 +15,9 @@ import ( // Config contains the configuration options for the loader type Config struct { Templates []string + TemplateURLs []string Workflows []string + WorkflowURLs []string ExcludeTemplates []string IncludeTemplates []string @@ -37,6 +39,7 @@ type Store struct { pathFilter *filter.PathFilter config *Config finalTemplates []string + finalWorkflows []string templates []*templates.Template workflows []*templates.Template @@ -61,13 +64,24 @@ func New(config *Config) (*Store, error) { IncludedTemplates: config.IncludeTemplates, ExcludedTemplates: config.ExcludeTemplates, }, config.Catalog), + finalTemplates: config.Templates, + finalWorkflows: config.Workflows, + } + + if len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 { + remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs) + if err != nil { + return store, err + } + store.finalTemplates = append(store.finalTemplates, remoteTemplates...) + store.finalWorkflows = append(store.finalWorkflows, remoteWorkflows...) } // Handle a case with no templates or workflows, where we use base directory - if len(config.Templates) == 0 && len(config.Workflows) == 0 { - config.Templates = append(config.Templates, config.TemplatesDirectory) + if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 { + store.finalTemplates = []string{config.TemplatesDirectory} } - store.finalTemplates = append(store.finalTemplates, config.Templates...) + return store, nil } @@ -90,7 +104,7 @@ func (store *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) { // the complete compiled templates for a nuclei execution configuration. func (store *Store) Load() { store.templates = store.LoadTemplates(store.finalTemplates) - store.workflows = store.LoadWorkflows(store.config.Workflows) + store.workflows = store.LoadWorkflows(store.finalWorkflows) } // ValidateTemplates takes a list of templates and validates them diff --git a/v2/pkg/catalog/loader/remote_loader.go b/v2/pkg/catalog/loader/remote_loader.go new file mode 100644 index 00000000..d8f71eda --- /dev/null +++ b/v2/pkg/catalog/loader/remote_loader.go @@ -0,0 +1,94 @@ +package loader + +import ( + "bufio" + "fmt" + "github.com/pkg/errors" + "net/http" + "strings" +) + +type ContentType string + +const ( + Template ContentType = "Template" + Workflow ContentType = "Workflow" +) + +type RemoteContentError struct { + Content []string + Type ContentType + Error error +} + +func getRemoteTemplatesAndWorkflows(templateURLs []string, workflowURLs []string) ([]string, []string, error) { + remoteContentErrorChannel := make(chan RemoteContentError) + + for _, templateURL := range templateURLs { + go getRemoteContent(templateURL, remoteContentErrorChannel, Template) + } + for _, workflowURL := range workflowURLs { + go getRemoteContent(workflowURL, remoteContentErrorChannel, Workflow) + } + + var remoteTemplateList []string + var remoteWorkFlowList []string + var err error + for i := 0; i < (len(templateURLs) + len(workflowURLs)); i++ { + remoteContentError := <-remoteContentErrorChannel + if remoteContentError.Error != nil { + if err != nil { + err = errors.New(remoteContentError.Error.Error() + ": " + err.Error()) + } else { + err = remoteContentError.Error + } + } else { + if remoteContentError.Type == Template { + remoteTemplateList = append(remoteTemplateList, remoteContentError.Content...) + } else if remoteContentError.Type == Workflow { + remoteWorkFlowList = append(remoteWorkFlowList, remoteContentError.Content...) + } + } + } + + return remoteTemplateList, remoteWorkFlowList, err +} + +func getRemoteContent(URL string, w chan<- RemoteContentError, contentType ContentType) { + response, err := http.Get(URL) + if err != nil { + w <- RemoteContentError{ + Error: err, + } + return + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode > 299 { + w <- RemoteContentError{ + Error: fmt.Errorf("get \"%s\": unexpect status %d", URL, response.StatusCode), + } + return + } + + scanner := bufio.NewScanner(response.Body) + var templateList []string + for scanner.Scan() { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + templateList = append(templateList, text) + } + + if err := scanner.Err(); err != nil { + w <- RemoteContentError{ + Error: errors.Wrap(err, "get \"%s\""), + } + return + } + + w <- RemoteContentError{ + Content: templateList, + Type: contentType, + } +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 0fd55b2c..c19ad18a 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -15,8 +15,12 @@ type Options struct { ExcludeTags goflags.NormalizedStringSlice // Workflows specifies any workflows to run by nuclei Workflows goflags.StringSlice + // WorkflowURLs specifies URLs to a list of workflows to use + WorkflowURLs goflags.StringSlice // Templates specifies the template/templates to use Templates goflags.StringSlice + // TemplateURLs specifies URLs to a list of templates to use + TemplateURLs goflags.StringSlice // ExcludedTemplates specifies the template/templates to exclude ExcludedTemplates goflags.StringSlice // CustomHeaders is the list of custom global headers to send with each request.