2020-04-03 21:20:32 +00:00
|
|
|
package templates
|
|
|
|
|
|
|
|
import (
|
2020-05-05 19:42:28 +00:00
|
|
|
"fmt"
|
2021-02-16 09:43:01 +00:00
|
|
|
"io/ioutil"
|
2020-04-03 21:20:32 +00:00
|
|
|
"os"
|
2021-08-27 18:45:28 +00:00
|
|
|
"regexp"
|
2021-02-03 19:39:29 +00:00
|
|
|
"strings"
|
2020-04-03 21:20:32 +00:00
|
|
|
|
2020-12-29 10:08:14 +00:00
|
|
|
"github.com/pkg/errors"
|
2021-07-19 18:04:08 +00:00
|
|
|
"gopkg.in/yaml.v2"
|
|
|
|
|
2021-08-27 18:45:28 +00:00
|
|
|
"github.com/projectdiscovery/gologger"
|
2021-02-05 19:06:43 +00:00
|
|
|
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
|
2020-12-29 10:08:14 +00:00
|
|
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
2021-01-01 21:09:27 +00:00
|
|
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/executer"
|
2021-02-05 19:06:43 +00:00
|
|
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/offlinehttp"
|
2021-08-27 18:45:28 +00:00
|
|
|
"github.com/projectdiscovery/nuclei/v2/pkg/templates/cache"
|
2021-07-19 18:04:08 +00:00
|
|
|
"github.com/projectdiscovery/nuclei/v2/pkg/utils"
|
2020-04-03 21:20:32 +00:00
|
|
|
)
|
|
|
|
|
2021-08-27 18:51:07 +00:00
|
|
|
var (
|
|
|
|
ErrCreateTemplateExecutor = errors.New("cannot create template executer")
|
|
|
|
parsedTemplatesCache = cache.New()
|
|
|
|
fieldErrorRegexp = regexp.MustCompile(`not found in`)
|
|
|
|
)
|
2021-08-27 14:06:06 +00:00
|
|
|
|
2020-06-29 12:13:08 +00:00
|
|
|
// Parse parses a yaml request template file
|
2021-02-26 07:43:11 +00:00
|
|
|
//nolint:gocritic // this cannot be passed by pointer
|
2021-08-27 14:06:06 +00:00
|
|
|
// TODO make sure reading from the disk the template parsing happens once: see parsers.ParseTemplate vs templates.Parse
|
2021-07-21 05:32:44 +00:00
|
|
|
func Parse(filePath string, preprocessor Preprocessor, options protocols.ExecuterOptions) (*Template, error) {
|
2021-08-27 18:45:28 +00:00
|
|
|
if value, err := parsedTemplatesCache.Has(filePath); value != nil {
|
|
|
|
return value.(*Template), err
|
2021-08-27 14:06:06 +00:00
|
|
|
}
|
|
|
|
|
2020-04-03 21:20:32 +00:00
|
|
|
template := &Template{}
|
|
|
|
|
2021-01-01 09:58:28 +00:00
|
|
|
f, err := os.Open(filePath)
|
2020-04-03 21:20:32 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2021-02-16 09:43:01 +00:00
|
|
|
defer f.Close()
|
2020-04-03 21:20:32 +00:00
|
|
|
|
2021-02-16 09:43:01 +00:00
|
|
|
data, err := ioutil.ReadAll(f)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
data = template.expandPreprocessors(data)
|
2021-07-21 05:32:44 +00:00
|
|
|
if preprocessor != nil {
|
|
|
|
data = preprocessor.Process(data)
|
|
|
|
}
|
|
|
|
|
2021-08-27 18:45:28 +00:00
|
|
|
err = yaml.UnmarshalStrict(data, template)
|
2020-04-03 21:20:32 +00:00
|
|
|
if err != nil {
|
2021-08-27 18:45:28 +00:00
|
|
|
if !fieldErrorRegexp.MatchString(err.Error()) {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
gologger.Warning().Msgf("Unrecognized fields in template %s: %s", filePath, err)
|
2020-04-03 21:20:32 +00:00
|
|
|
}
|
2020-06-26 12:37:55 +00:00
|
|
|
|
2021-08-03 11:51:34 +00:00
|
|
|
if utils.IsBlank(template.Info.Name) {
|
2021-02-03 19:39:29 +00:00
|
|
|
return nil, errors.New("no template name field provided")
|
|
|
|
}
|
2021-08-03 11:51:34 +00:00
|
|
|
if template.Info.Authors.IsEmpty() {
|
2021-02-03 19:39:29 +00:00
|
|
|
return nil, errors.New("no template author field provided")
|
|
|
|
}
|
|
|
|
|
2020-12-29 10:08:14 +00:00
|
|
|
// Setting up variables regarding template metadata
|
|
|
|
options.TemplateID = template.ID
|
|
|
|
options.TemplateInfo = template.Info
|
2021-01-01 09:58:28 +00:00
|
|
|
options.TemplatePath = filePath
|
2020-07-31 15:13:51 +00:00
|
|
|
|
2020-06-29 12:13:08 +00:00
|
|
|
// If no requests, and it is also not a workflow, return error.
|
2021-08-03 11:51:34 +00:00
|
|
|
if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows) == 0 {
|
2020-08-26 18:05:31 +00:00
|
|
|
return nil, fmt.Errorf("no requests defined for %s", template.ID)
|
2020-06-26 12:37:55 +00:00
|
|
|
}
|
2020-04-03 21:20:32 +00:00
|
|
|
|
2020-12-29 12:32:45 +00:00
|
|
|
// Compile the workflow request
|
2021-08-03 11:51:34 +00:00
|
|
|
if len(template.Workflows) > 0 {
|
2020-12-30 07:56:55 +00:00
|
|
|
compiled := &template.Workflow
|
2021-07-04 23:05:53 +00:00
|
|
|
|
2021-08-03 12:26:45 +00:00
|
|
|
compileWorkflow(preprocessor, &options, compiled, options.WorkflowLoader)
|
2020-12-30 07:56:55 +00:00
|
|
|
template.CompiledWorkflow = compiled
|
2021-02-23 17:25:29 +00:00
|
|
|
template.CompiledWorkflow.Options = &options
|
2020-12-29 12:32:45 +00:00
|
|
|
}
|
|
|
|
|
2020-12-29 10:08:14 +00:00
|
|
|
// Compile the requests found
|
2021-01-01 21:09:27 +00:00
|
|
|
requests := []protocols.Request{}
|
2021-02-07 09:42:38 +00:00
|
|
|
if len(template.RequestsDNS) > 0 && !options.Options.OfflineHTTP {
|
2021-01-01 21:09:27 +00:00
|
|
|
for _, req := range template.RequestsDNS {
|
|
|
|
requests = append(requests, req)
|
|
|
|
}
|
2021-02-05 19:06:43 +00:00
|
|
|
template.Executer = executer.NewExecuter(requests, &options)
|
2020-12-29 11:03:25 +00:00
|
|
|
}
|
|
|
|
if len(template.RequestsHTTP) > 0 {
|
2021-02-05 19:06:43 +00:00
|
|
|
if options.Options.OfflineHTTP {
|
2021-02-26 07:43:11 +00:00
|
|
|
operatorsList := []*operators.Operators{}
|
2021-02-05 19:06:43 +00:00
|
|
|
|
2021-03-04 13:44:53 +00:00
|
|
|
mainLoop:
|
2021-02-05 19:06:43 +00:00
|
|
|
for _, req := range template.RequestsHTTP {
|
2021-03-04 13:44:53 +00:00
|
|
|
for _, path := range req.Path {
|
|
|
|
if !(strings.EqualFold(path, "{{BaseURL}}") || strings.EqualFold(path, "{{BaseURL}}/")) {
|
|
|
|
break mainLoop
|
|
|
|
}
|
|
|
|
}
|
2021-02-26 07:43:11 +00:00
|
|
|
operatorsList = append(operatorsList, &req.Operators)
|
2021-02-05 19:06:43 +00:00
|
|
|
}
|
2021-03-04 13:44:53 +00:00
|
|
|
if len(operatorsList) > 0 {
|
|
|
|
options.Operators = operatorsList
|
|
|
|
template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options)
|
|
|
|
}
|
2021-02-05 19:06:43 +00:00
|
|
|
} else {
|
|
|
|
for _, req := range template.RequestsHTTP {
|
|
|
|
requests = append(requests, req)
|
|
|
|
}
|
|
|
|
template.Executer = executer.NewExecuter(requests, &options)
|
2021-01-01 21:09:27 +00:00
|
|
|
}
|
2021-01-01 09:58:28 +00:00
|
|
|
}
|
2021-02-07 09:42:38 +00:00
|
|
|
if len(template.RequestsFile) > 0 && !options.Options.OfflineHTTP {
|
2021-01-01 21:09:27 +00:00
|
|
|
for _, req := range template.RequestsFile {
|
|
|
|
requests = append(requests, req)
|
|
|
|
}
|
2021-02-05 19:06:43 +00:00
|
|
|
template.Executer = executer.NewExecuter(requests, &options)
|
2020-12-29 11:03:25 +00:00
|
|
|
}
|
2021-02-07 09:42:38 +00:00
|
|
|
if len(template.RequestsNetwork) > 0 && !options.Options.OfflineHTTP {
|
2021-01-01 21:09:27 +00:00
|
|
|
for _, req := range template.RequestsNetwork {
|
|
|
|
requests = append(requests, req)
|
|
|
|
}
|
2021-02-05 19:06:43 +00:00
|
|
|
template.Executer = executer.NewExecuter(requests, &options)
|
2020-12-30 09:24:20 +00:00
|
|
|
}
|
2021-02-21 11:01:34 +00:00
|
|
|
if len(template.RequestsHeadless) > 0 && !options.Options.OfflineHTTP && options.Options.Headless {
|
|
|
|
for _, req := range template.RequestsHeadless {
|
|
|
|
requests = append(requests, req)
|
|
|
|
}
|
|
|
|
template.Executer = executer.NewExecuter(requests, &options)
|
|
|
|
}
|
2021-01-01 14:06:21 +00:00
|
|
|
if template.Executer != nil {
|
|
|
|
err := template.Executer.Compile()
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrap(err, "could not compile request")
|
|
|
|
}
|
2021-01-11 20:30:11 +00:00
|
|
|
template.TotalRequests += template.Executer.Requests()
|
2020-12-29 11:03:25 +00:00
|
|
|
}
|
2021-02-22 07:34:42 +00:00
|
|
|
if template.Executer == nil && template.CompiledWorkflow == nil {
|
2021-08-27 18:51:07 +00:00
|
|
|
return nil, ErrCreateTemplateExecutor
|
2021-02-07 09:42:38 +00:00
|
|
|
}
|
2021-06-05 17:30:59 +00:00
|
|
|
template.Path = filePath
|
2021-08-27 14:06:06 +00:00
|
|
|
|
2021-08-27 18:45:28 +00:00
|
|
|
parsedTemplatesCache.Store(filePath, template, err)
|
2020-04-03 21:20:32 +00:00
|
|
|
return template, nil
|
|
|
|
}
|