Merge pull request #1123 from EndPositive/remote-template-workflow-lists

Remote template workflow lists
dev
Ice3man 2021-11-02 16:12:29 +05:30 committed by GitHub
commit 09cad2557c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 316 additions and 9 deletions

View File

@ -0,0 +1,10 @@
id: workflow-example
info:
name: Test Workflow Template
author: pdteam
severity: info
workflows:
- template: workflow/match-1.yaml
- template: workflow/match-2.yaml

View File

@ -0,0 +1,11 @@
id: condition-matched-workflow
info:
name: Condition Matched Workflow
author: pdteam
severity: info
workflows:
- template: workflow/match-1.yaml
subtemplates:
- template: workflow/match-2.yaml

View File

@ -0,0 +1,17 @@
id: basic-get-headers
info:
name: Basic GET Headers Request
author: pdteam
severity: info
requests:
- method: GET
path:
- "{{BaseURL}}"
headers:
test: nuclei
matchers:
- type: word
words:
- "This is test headers matcher text"

View File

@ -0,0 +1,15 @@
id: basic-get
info:
name: Basic GET Request
author: pdteam
severity: info
requests:
- method: GET
path:
- "{{BaseURL}}"
matchers:
- type: word
words:
- "This is test matcher text"

View File

@ -0,0 +1,2 @@
loader/get.yaml
loader/get-headers.yaml

View File

@ -0,0 +1,2 @@
loader/basic.yaml
loader/condition-matched.yaml

View File

@ -26,6 +26,7 @@ func main() {
"network": networkTestcases,
"dns": dnsTestCases,
"workflow": workflowTestcases,
"loader": loaderTestcases,
}
for proto, tests := range protocolTests {
if protocol == "" || protocol == proto {

View File

@ -0,0 +1,122 @@
package main
import (
"fmt"
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"net/http"
"net/http/httptest"
"os"
"strings"
)
var loaderTestcases = map[string]testutils.TestCase{
"loader/template-list.yaml": &remoteTemplateList{},
"loader/workflow-list.yaml": &remoteWorkflowList{},
"loader/nonexistent-template-list.yaml": &nonExistentTemplateList{},
"loader/nonexistent-workflow-list.yaml": &nonExistentWorkflowList{},
}
type remoteTemplateList struct{}
// Execute executes a test case and returns an error if occurred
func (h *remoteTemplateList) Execute(templateList string) error {
router := httprouter.New()
router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
fmt.Fprintf(w, "This is test matcher text")
if strings.EqualFold(r.Header.Get("test"), "nuclei") {
fmt.Fprintf(w, "This is test headers matcher text")
}
}))
router.GET("/template_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
file, err := os.ReadFile(templateList)
if err != nil {
w.WriteHeader(500)
}
_, err = w.Write(file)
if err != nil {
w.WriteHeader(500)
}
}))
ts := httptest.NewServer(router)
defer ts.Close()
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list")
if err != nil {
return err
}
if len(results) != 2 {
return errIncorrectResultsCount(results)
}
return nil
}
type remoteWorkflowList struct{}
// Execute executes a test case and returns an error if occurred
func (h *remoteWorkflowList) Execute(workflowList string) error {
router := httprouter.New()
router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
fmt.Fprintf(w, "This is test matcher text")
if strings.EqualFold(r.Header.Get("test"), "nuclei") {
fmt.Fprintf(w, "This is test headers matcher text")
}
}))
router.GET("/workflow_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
file, err := os.ReadFile(workflowList)
if err != nil {
w.WriteHeader(500)
}
_, err = w.Write(file)
if err != nil {
w.WriteHeader(500)
}
}))
ts := httptest.NewServer(router)
defer ts.Close()
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list")
if err != nil {
return err
}
if len(results) != 3 {
return errIncorrectResultsCount(results)
}
return nil
}
type nonExistentTemplateList struct{}
// Execute executes a test case and returns an error if occurred
func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error {
router := httprouter.New()
ts := httptest.NewServer(router)
defer ts.Close()
_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/404")
if err == nil {
return fmt.Errorf("expected error for nonexisting workflow url")
}
return nil
}
type nonExistentWorkflowList struct{}
// Execute executes a test case and returns an error if occurred
func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error {
router := httprouter.New()
ts := httptest.NewServer(router)
defer ts.Close()
_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/404")
if err == nil {
return fmt.Errorf("expected error for nonexisting workflow url")
}
return nil
}

View File

@ -54,8 +54,10 @@ on extensive configurability, massive extensibility and ease of use.`)
createGroup(flagSet, "templates", "Templates",
flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"),
flagSet.StringSliceVarP(&options.TemplateURLs, "template-urls", "tu", []string{}, "URL to a list of templates"),
flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"),
flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "workflow or workflow directory paths to include in the scan"),
flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-urls", "wu", []string{}, "URL to a list of workflows to run"),
flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
)

View File

@ -366,7 +366,9 @@ func (r *Runner) RunEnumeration() error {
loaderConfig := loader.Config{
Templates: r.options.Templates,
TemplateURLs: r.options.TemplateURLs,
Workflows: r.options.Workflows,
WorkflowURLs: r.options.WorkflowURLs,
ExcludeTemplates: r.options.ExcludedTemplates,
Tags: r.options.Tags,
ExcludeTags: r.options.ExcludeTags,

View File

@ -12,15 +12,15 @@ import (
// RunNucleiTemplateAndGetResults returns a list of results for a template
func RunNucleiTemplateAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
return runNucleiAndGetResults(true, template, url, debug, extra...)
return RunNucleiAndGetResults(true, template, url, debug, extra...)
}
// RunNucleiWorkflowAndGetResults returns a list of results for a workflow
func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
return runNucleiAndGetResults(false, template, url, debug, extra...)
return RunNucleiAndGetResults(false, template, url, debug, extra...)
}
func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) {
func RunNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) {
var templateOrWorkflowFlag string
if isTemplate {
templateOrWorkflowFlag = "-t"
@ -28,11 +28,22 @@ func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, e
templateOrWorkflowFlag = "-w"
}
cmd := exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-silent")
return RunNucleiBareArgsAndGetResults(debug, append([]string{
templateOrWorkflowFlag,
template,
"-target",
url,
}, extra...)...)
}
func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, error) {
cmd := exec.Command("./nuclei")
if debug {
cmd = exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-debug")
cmd.Args = append(cmd.Args, "-debug")
cmd.Stderr = os.Stderr
fmt.Println(cmd.String())
} else {
cmd.Args = append(cmd.Args, "-silent")
}
cmd.Args = append(cmd.Args, extra...)
data, err := cmd.Output()

View File

@ -15,7 +15,9 @@ import (
// Config contains the configuration options for the loader
type Config struct {
Templates []string
TemplateURLs []string
Workflows []string
WorkflowURLs []string
ExcludeTemplates []string
IncludeTemplates []string
@ -37,6 +39,7 @@ type Store struct {
pathFilter *filter.PathFilter
config *Config
finalTemplates []string
finalWorkflows []string
templates []*templates.Template
workflows []*templates.Template
@ -61,13 +64,24 @@ func New(config *Config) (*Store, error) {
IncludedTemplates: config.IncludeTemplates,
ExcludedTemplates: config.ExcludeTemplates,
}, config.Catalog),
finalTemplates: config.Templates,
finalWorkflows: config.Workflows,
}
if len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 {
remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs)
if err != nil {
return store, err
}
store.finalTemplates = append(store.finalTemplates, remoteTemplates...)
store.finalWorkflows = append(store.finalWorkflows, remoteWorkflows...)
}
// Handle a case with no templates or workflows, where we use base directory
if len(config.Templates) == 0 && len(config.Workflows) == 0 {
config.Templates = append(config.Templates, config.TemplatesDirectory)
if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 {
store.finalTemplates = []string{config.TemplatesDirectory}
}
store.finalTemplates = append(store.finalTemplates, config.Templates...)
return store, nil
}
@ -90,7 +104,7 @@ func (store *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
// the complete compiled templates for a nuclei execution configuration.
func (store *Store) Load() {
store.templates = store.LoadTemplates(store.finalTemplates)
store.workflows = store.LoadWorkflows(store.config.Workflows)
store.workflows = store.LoadWorkflows(store.finalWorkflows)
}
// ValidateTemplates takes a list of templates and validates them

View File

@ -0,0 +1,94 @@
package loader
import (
"bufio"
"fmt"
"github.com/pkg/errors"
"net/http"
"strings"
)
type ContentType string
const (
Template ContentType = "Template"
Workflow ContentType = "Workflow"
)
type RemoteContentError struct {
Content []string
Type ContentType
Error error
}
func getRemoteTemplatesAndWorkflows(templateURLs []string, workflowURLs []string) ([]string, []string, error) {
remoteContentErrorChannel := make(chan RemoteContentError)
for _, templateURL := range templateURLs {
go getRemoteContent(templateURL, remoteContentErrorChannel, Template)
}
for _, workflowURL := range workflowURLs {
go getRemoteContent(workflowURL, remoteContentErrorChannel, Workflow)
}
var remoteTemplateList []string
var remoteWorkFlowList []string
var err error
for i := 0; i < (len(templateURLs) + len(workflowURLs)); i++ {
remoteContentError := <-remoteContentErrorChannel
if remoteContentError.Error != nil {
if err != nil {
err = errors.New(remoteContentError.Error.Error() + ": " + err.Error())
} else {
err = remoteContentError.Error
}
} else {
if remoteContentError.Type == Template {
remoteTemplateList = append(remoteTemplateList, remoteContentError.Content...)
} else if remoteContentError.Type == Workflow {
remoteWorkFlowList = append(remoteWorkFlowList, remoteContentError.Content...)
}
}
}
return remoteTemplateList, remoteWorkFlowList, err
}
func getRemoteContent(URL string, w chan<- RemoteContentError, contentType ContentType) {
response, err := http.Get(URL)
if err != nil {
w <- RemoteContentError{
Error: err,
}
return
}
defer response.Body.Close()
if response.StatusCode < 200 || response.StatusCode > 299 {
w <- RemoteContentError{
Error: fmt.Errorf("get \"%s\": unexpect status %d", URL, response.StatusCode),
}
return
}
scanner := bufio.NewScanner(response.Body)
var templateList []string
for scanner.Scan() {
text := strings.TrimSpace(scanner.Text())
if text == "" {
continue
}
templateList = append(templateList, text)
}
if err := scanner.Err(); err != nil {
w <- RemoteContentError{
Error: errors.Wrap(err, "get \"%s\""),
}
return
}
w <- RemoteContentError{
Content: templateList,
Type: contentType,
}
}

View File

@ -15,8 +15,12 @@ type Options struct {
ExcludeTags goflags.NormalizedStringSlice
// Workflows specifies any workflows to run by nuclei
Workflows goflags.StringSlice
// WorkflowURLs specifies URLs to a list of workflows to use
WorkflowURLs goflags.StringSlice
// Templates specifies the template/templates to use
Templates goflags.StringSlice
// TemplateURLs specifies URLs to a list of templates to use
TemplateURLs goflags.StringSlice
// ExcludedTemplates specifies the template/templates to exclude
ExcludedTemplates goflags.StringSlice
// CustomHeaders is the list of custom global headers to send with each request.