mirror of https://github.com/daffainfo/nuclei.git
Merge pull request #1123 from EndPositive/remote-template-workflow-lists
Remote template workflow listsdev
commit
09cad2557c
|
@ -0,0 +1,10 @@
|
|||
id: workflow-example
|
||||
|
||||
info:
|
||||
name: Test Workflow Template
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
workflows:
|
||||
- template: workflow/match-1.yaml
|
||||
- template: workflow/match-2.yaml
|
|
@ -0,0 +1,11 @@
|
|||
id: condition-matched-workflow
|
||||
|
||||
info:
|
||||
name: Condition Matched Workflow
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
workflows:
|
||||
- template: workflow/match-1.yaml
|
||||
subtemplates:
|
||||
- template: workflow/match-2.yaml
|
|
@ -0,0 +1,17 @@
|
|||
id: basic-get-headers
|
||||
|
||||
info:
|
||||
name: Basic GET Headers Request
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
requests:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
headers:
|
||||
test: nuclei
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "This is test headers matcher text"
|
|
@ -0,0 +1,15 @@
|
|||
id: basic-get
|
||||
|
||||
info:
|
||||
name: Basic GET Request
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
requests:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "This is test matcher text"
|
|
@ -0,0 +1,2 @@
|
|||
loader/get.yaml
|
||||
loader/get-headers.yaml
|
|
@ -0,0 +1,2 @@
|
|||
loader/basic.yaml
|
||||
loader/condition-matched.yaml
|
|
@ -26,6 +26,7 @@ func main() {
|
|||
"network": networkTestcases,
|
||||
"dns": dnsTestCases,
|
||||
"workflow": workflowTestcases,
|
||||
"loader": loaderTestcases,
|
||||
}
|
||||
for proto, tests := range protocolTests {
|
||||
if protocol == "" || protocol == proto {
|
||||
|
|
|
@ -0,0 +1,122 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/julienschmidt/httprouter"
|
||||
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var loaderTestcases = map[string]testutils.TestCase{
|
||||
"loader/template-list.yaml": &remoteTemplateList{},
|
||||
"loader/workflow-list.yaml": &remoteWorkflowList{},
|
||||
"loader/nonexistent-template-list.yaml": &nonExistentTemplateList{},
|
||||
"loader/nonexistent-workflow-list.yaml": &nonExistentWorkflowList{},
|
||||
}
|
||||
|
||||
type remoteTemplateList struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *remoteTemplateList) Execute(templateList string) error {
|
||||
router := httprouter.New()
|
||||
|
||||
router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
fmt.Fprintf(w, "This is test matcher text")
|
||||
if strings.EqualFold(r.Header.Get("test"), "nuclei") {
|
||||
fmt.Fprintf(w, "This is test headers matcher text")
|
||||
}
|
||||
}))
|
||||
|
||||
router.GET("/template_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
file, err := os.ReadFile(templateList)
|
||||
if err != nil {
|
||||
w.WriteHeader(500)
|
||||
}
|
||||
_, err = w.Write(file)
|
||||
if err != nil {
|
||||
w.WriteHeader(500)
|
||||
}
|
||||
}))
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(results) != 2 {
|
||||
return errIncorrectResultsCount(results)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type remoteWorkflowList struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *remoteWorkflowList) Execute(workflowList string) error {
|
||||
router := httprouter.New()
|
||||
|
||||
router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
fmt.Fprintf(w, "This is test matcher text")
|
||||
if strings.EqualFold(r.Header.Get("test"), "nuclei") {
|
||||
fmt.Fprintf(w, "This is test headers matcher text")
|
||||
}
|
||||
}))
|
||||
|
||||
router.GET("/workflow_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
file, err := os.ReadFile(workflowList)
|
||||
if err != nil {
|
||||
w.WriteHeader(500)
|
||||
}
|
||||
_, err = w.Write(file)
|
||||
if err != nil {
|
||||
w.WriteHeader(500)
|
||||
}
|
||||
}))
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(results) != 3 {
|
||||
return errIncorrectResultsCount(results)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type nonExistentTemplateList struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error {
|
||||
router := httprouter.New()
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/404")
|
||||
if err == nil {
|
||||
return fmt.Errorf("expected error for nonexisting workflow url")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type nonExistentWorkflowList struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error {
|
||||
router := httprouter.New()
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/404")
|
||||
if err == nil {
|
||||
return fmt.Errorf("expected error for nonexisting workflow url")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
|
@ -54,8 +54,10 @@ on extensive configurability, massive extensibility and ease of use.`)
|
|||
|
||||
createGroup(flagSet, "templates", "Templates",
|
||||
flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"),
|
||||
flagSet.StringSliceVarP(&options.TemplateURLs, "template-urls", "tu", []string{}, "URL to a list of templates"),
|
||||
flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"),
|
||||
flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "workflow or workflow directory paths to include in the scan"),
|
||||
flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-urls", "wu", []string{}, "URL to a list of workflows to run"),
|
||||
flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
|
||||
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
|
||||
)
|
||||
|
|
|
@ -366,7 +366,9 @@ func (r *Runner) RunEnumeration() error {
|
|||
|
||||
loaderConfig := loader.Config{
|
||||
Templates: r.options.Templates,
|
||||
TemplateURLs: r.options.TemplateURLs,
|
||||
Workflows: r.options.Workflows,
|
||||
WorkflowURLs: r.options.WorkflowURLs,
|
||||
ExcludeTemplates: r.options.ExcludedTemplates,
|
||||
Tags: r.options.Tags,
|
||||
ExcludeTags: r.options.ExcludeTags,
|
||||
|
|
|
@ -12,15 +12,15 @@ import (
|
|||
|
||||
// RunNucleiTemplateAndGetResults returns a list of results for a template
|
||||
func RunNucleiTemplateAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
|
||||
return runNucleiAndGetResults(true, template, url, debug, extra...)
|
||||
return RunNucleiAndGetResults(true, template, url, debug, extra...)
|
||||
}
|
||||
|
||||
// RunNucleiWorkflowAndGetResults returns a list of results for a workflow
|
||||
func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
|
||||
return runNucleiAndGetResults(false, template, url, debug, extra...)
|
||||
return RunNucleiAndGetResults(false, template, url, debug, extra...)
|
||||
}
|
||||
|
||||
func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) {
|
||||
func RunNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) {
|
||||
var templateOrWorkflowFlag string
|
||||
if isTemplate {
|
||||
templateOrWorkflowFlag = "-t"
|
||||
|
@ -28,11 +28,22 @@ func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, e
|
|||
templateOrWorkflowFlag = "-w"
|
||||
}
|
||||
|
||||
cmd := exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-silent")
|
||||
return RunNucleiBareArgsAndGetResults(debug, append([]string{
|
||||
templateOrWorkflowFlag,
|
||||
template,
|
||||
"-target",
|
||||
url,
|
||||
}, extra...)...)
|
||||
}
|
||||
|
||||
func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, error) {
|
||||
cmd := exec.Command("./nuclei")
|
||||
if debug {
|
||||
cmd = exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-debug")
|
||||
cmd.Args = append(cmd.Args, "-debug")
|
||||
cmd.Stderr = os.Stderr
|
||||
fmt.Println(cmd.String())
|
||||
} else {
|
||||
cmd.Args = append(cmd.Args, "-silent")
|
||||
}
|
||||
cmd.Args = append(cmd.Args, extra...)
|
||||
data, err := cmd.Output()
|
||||
|
|
|
@ -15,7 +15,9 @@ import (
|
|||
// Config contains the configuration options for the loader
|
||||
type Config struct {
|
||||
Templates []string
|
||||
TemplateURLs []string
|
||||
Workflows []string
|
||||
WorkflowURLs []string
|
||||
ExcludeTemplates []string
|
||||
IncludeTemplates []string
|
||||
|
||||
|
@ -37,6 +39,7 @@ type Store struct {
|
|||
pathFilter *filter.PathFilter
|
||||
config *Config
|
||||
finalTemplates []string
|
||||
finalWorkflows []string
|
||||
|
||||
templates []*templates.Template
|
||||
workflows []*templates.Template
|
||||
|
@ -61,13 +64,24 @@ func New(config *Config) (*Store, error) {
|
|||
IncludedTemplates: config.IncludeTemplates,
|
||||
ExcludedTemplates: config.ExcludeTemplates,
|
||||
}, config.Catalog),
|
||||
finalTemplates: config.Templates,
|
||||
finalWorkflows: config.Workflows,
|
||||
}
|
||||
|
||||
if len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 {
|
||||
remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs)
|
||||
if err != nil {
|
||||
return store, err
|
||||
}
|
||||
store.finalTemplates = append(store.finalTemplates, remoteTemplates...)
|
||||
store.finalWorkflows = append(store.finalWorkflows, remoteWorkflows...)
|
||||
}
|
||||
|
||||
// Handle a case with no templates or workflows, where we use base directory
|
||||
if len(config.Templates) == 0 && len(config.Workflows) == 0 {
|
||||
config.Templates = append(config.Templates, config.TemplatesDirectory)
|
||||
if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 {
|
||||
store.finalTemplates = []string{config.TemplatesDirectory}
|
||||
}
|
||||
store.finalTemplates = append(store.finalTemplates, config.Templates...)
|
||||
|
||||
return store, nil
|
||||
}
|
||||
|
||||
|
@ -90,7 +104,7 @@ func (store *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
|
|||
// the complete compiled templates for a nuclei execution configuration.
|
||||
func (store *Store) Load() {
|
||||
store.templates = store.LoadTemplates(store.finalTemplates)
|
||||
store.workflows = store.LoadWorkflows(store.config.Workflows)
|
||||
store.workflows = store.LoadWorkflows(store.finalWorkflows)
|
||||
}
|
||||
|
||||
// ValidateTemplates takes a list of templates and validates them
|
||||
|
|
|
@ -0,0 +1,94 @@
|
|||
package loader
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"github.com/pkg/errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type ContentType string
|
||||
|
||||
const (
|
||||
Template ContentType = "Template"
|
||||
Workflow ContentType = "Workflow"
|
||||
)
|
||||
|
||||
type RemoteContentError struct {
|
||||
Content []string
|
||||
Type ContentType
|
||||
Error error
|
||||
}
|
||||
|
||||
func getRemoteTemplatesAndWorkflows(templateURLs []string, workflowURLs []string) ([]string, []string, error) {
|
||||
remoteContentErrorChannel := make(chan RemoteContentError)
|
||||
|
||||
for _, templateURL := range templateURLs {
|
||||
go getRemoteContent(templateURL, remoteContentErrorChannel, Template)
|
||||
}
|
||||
for _, workflowURL := range workflowURLs {
|
||||
go getRemoteContent(workflowURL, remoteContentErrorChannel, Workflow)
|
||||
}
|
||||
|
||||
var remoteTemplateList []string
|
||||
var remoteWorkFlowList []string
|
||||
var err error
|
||||
for i := 0; i < (len(templateURLs) + len(workflowURLs)); i++ {
|
||||
remoteContentError := <-remoteContentErrorChannel
|
||||
if remoteContentError.Error != nil {
|
||||
if err != nil {
|
||||
err = errors.New(remoteContentError.Error.Error() + ": " + err.Error())
|
||||
} else {
|
||||
err = remoteContentError.Error
|
||||
}
|
||||
} else {
|
||||
if remoteContentError.Type == Template {
|
||||
remoteTemplateList = append(remoteTemplateList, remoteContentError.Content...)
|
||||
} else if remoteContentError.Type == Workflow {
|
||||
remoteWorkFlowList = append(remoteWorkFlowList, remoteContentError.Content...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return remoteTemplateList, remoteWorkFlowList, err
|
||||
}
|
||||
|
||||
func getRemoteContent(URL string, w chan<- RemoteContentError, contentType ContentType) {
|
||||
response, err := http.Get(URL)
|
||||
if err != nil {
|
||||
w <- RemoteContentError{
|
||||
Error: err,
|
||||
}
|
||||
return
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode < 200 || response.StatusCode > 299 {
|
||||
w <- RemoteContentError{
|
||||
Error: fmt.Errorf("get \"%s\": unexpect status %d", URL, response.StatusCode),
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(response.Body)
|
||||
var templateList []string
|
||||
for scanner.Scan() {
|
||||
text := strings.TrimSpace(scanner.Text())
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
templateList = append(templateList, text)
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
w <- RemoteContentError{
|
||||
Error: errors.Wrap(err, "get \"%s\""),
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
w <- RemoteContentError{
|
||||
Content: templateList,
|
||||
Type: contentType,
|
||||
}
|
||||
}
|
|
@ -15,8 +15,12 @@ type Options struct {
|
|||
ExcludeTags goflags.NormalizedStringSlice
|
||||
// Workflows specifies any workflows to run by nuclei
|
||||
Workflows goflags.StringSlice
|
||||
// WorkflowURLs specifies URLs to a list of workflows to use
|
||||
WorkflowURLs goflags.StringSlice
|
||||
// Templates specifies the template/templates to use
|
||||
Templates goflags.StringSlice
|
||||
// TemplateURLs specifies URLs to a list of templates to use
|
||||
TemplateURLs goflags.StringSlice
|
||||
// ExcludedTemplates specifies the template/templates to exclude
|
||||
ExcludedTemplates goflags.StringSlice
|
||||
// CustomHeaders is the list of custom global headers to send with each request.
|
||||
|
|
Loading…
Reference in New Issue