daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into deserialization-helpers

dev
Ice3man 2021-07-24 19:20:36 +05:30 committed by GitHub
parent 0fd65e2bf2 2b735c2e1e
commit b78780cd96
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 131 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -1,4 +1,4 @@
FROM golang:1.15-alpine as build-env
FROM golang:1.16.6-alpine as build-env
RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
FROM alpine:latest

15
v2/pkg/catalog/loader/loader.go
View File

@ -38,6 +38,8 @@ type Store struct {
templates []*templates.Template
workflows []*templates.Template
preprocessor templates.Preprocessor
}
// New creates a new template store based on provided configuration
@ -76,6 +78,11 @@ func (s *Store) Workflows() []*templates.Template {
return s.workflows
}
// RegisterPreprocessor allows a custom preprocessor to be passed to the store to run against templates
func (s *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
s.preprocessor = preprocessor
}
// Load loads all the templates from a store, performs filtering and returns
// the complete compiled templates for a nuclei execution configuration.
func (s *Store) Load() {
@ -105,7 +112,7 @@ func (s *Store) ValidateTemplates(templatesList, workflowsList []string) bool {
gologger.Error().Msgf("Error occurred loading template %s: %s\n", k, err)
continue
}
_, err = templates.Parse(k, s.config.ExecutorOptions)
_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
if err != nil {
if strings.Contains(err.Error(), "cannot create template executer") {
continue
@ -129,7 +136,7 @@ func (s *Store) ValidateTemplates(templatesList, workflowsList []string) bool {
notErrored = false
gologger.Error().Msgf("Error occurred loading workflow %s: %s\n", k, err)
}
_, err = templates.Parse(k, s.config.ExecutorOptions)
_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
if err != nil {
if strings.Contains(err.Error(), "cannot create template executer") {
continue
@ -156,7 +163,7 @@ func (s *Store) LoadTemplates(templatesList []string) []*templates.Template {
gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
}
if loaded {
parsed, err := templates.Parse(k, s.config.ExecutorOptions)
parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
if err != nil {
gologger.Warning().Msgf("Could not parse template %s: %s\n", k, err)
} else if parsed != nil {
@ -179,7 +186,7 @@ func (s *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err)
}
if loaded {
parsed, err := templates.Parse(k, s.config.ExecutorOptions)
parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
if err != nil {
gologger.Warning().Msgf("Could not parse workflow %s: %s\n", k, err)
} else if parsed != nil {

193
v2/pkg/operators/common/dsl/dsl.go
View File

@ -6,6 +6,7 @@ import (
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"html"
"math"
@ -29,113 +30,88 @@ const (
withMaxRandArgsSize = withCutSetArgsSize
)
// HelperFunctions contains the dsl helper functions
func HelperFunctions() map[string]govaluate.ExpressionFunction {
functions := make(map[string]govaluate.ExpressionFunction)
functions["len"] = func(args ...interface{}) (interface{}, error) {
var functions = map[string]govaluate.ExpressionFunction{
"len": func(args ...interface{}) (interface{}, error) {
length := len(types.ToString(args[0]))
return float64(length), nil
}
functions["toupper"] = func(args ...interface{}) (interface{}, error) {
},
"toupper": func(args ...interface{}) (interface{}, error) {
return strings.ToUpper(types.ToString(args[0])), nil
}
functions["tolower"] = func(args ...interface{}) (interface{}, error) {
},
"tolower": func(args ...interface{}) (interface{}, error) {
return strings.ToLower(types.ToString(args[0])), nil
}
functions["replace"] = func(args ...interface{}) (interface{}, error) {
},
"replace": func(args ...interface{}) (interface{}, error) {
return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil
}
functions["replace_regex"] = func(args ...interface{}) (interface{}, error) {
},
"replace_regex": func(args ...interface{}) (interface{}, error) {
compiled, err := regexp.Compile(types.ToString(args[1]))
if err != nil {
return nil, err
}
return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil
}
functions["trim"] = func(args ...interface{}) (interface{}, error) {
},
"trim": func(args ...interface{}) (interface{}, error) {
return strings.Trim(types.ToString(args[0]), types.ToString(args[2])), nil
}
functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
},
"trimleft": func(args ...interface{}) (interface{}, error) {
return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil
}
functions["trimright"] = func(args ...interface{}) (interface{}, error) {
},
"trimright": func(args ...interface{}) (interface{}, error) {
return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil
}
functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
},
"trimspace": func(args ...interface{}) (interface{}, error) {
return strings.TrimSpace(types.ToString(args[0])), nil
}
functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
},
"trimprefix": func(args ...interface{}) (interface{}, error) {
return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil
}
functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
},
"trimsuffix": func(args ...interface{}) (interface{}, error) {
return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil
}
functions["reverse"] = func(args ...interface{}) (interface{}, error) {
},
"reverse": func(args ...interface{}) (interface{}, error) {
return reverseString(types.ToString(args[0])), nil
}
},
// encoding
functions["base64"] = func(args ...interface{}) (interface{}, error) {
"base64": func(args ...interface{}) (interface{}, error) {
sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
return sEnc, nil
}
},
// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
functions["base64_py"] = func(args ...interface{}) (interface{}, error) {
"base64_py": func(args ...interface{}) (interface{}, error) {
sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
return deserialization.InsertInto(sEnc, 76, '\n'), nil
}
functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
},
"base64_decode": func(args ...interface{}) (interface{}, error) {
return base64.StdEncoding.DecodeString(types.ToString(args[0]))
}
functions["url_encode"] = func(args ...interface{}) (interface{}, error) {
},
"url_encode": func(args ...interface{}) (interface{}, error) {
return url.PathEscape(types.ToString(args[0])), nil
}
functions["url_decode"] = func(args ...interface{}) (interface{}, error) {
},
"url_decode": func(args ...interface{}) (interface{}, error) {
return url.PathUnescape(types.ToString(args[0]))
}
functions["hex_encode"] = func(args ...interface{}) (interface{}, error) {
},
"hex_encode": func(args ...interface{}) (interface{}, error) {
return hex.EncodeToString([]byte(types.ToString(args[0]))), nil
}
functions["hex_decode"] = func(args ...interface{}) (interface{}, error) {
},
"hex_decode": func(args ...interface{}) (interface{}, error) {
hx, _ := hex.DecodeString(types.ToString(args[0]))
return string(hx), nil
}
functions["html_escape"] = func(args ...interface{}) (interface{}, error) {
},
"html_escape": func(args ...interface{}) (interface{}, error) {
return html.EscapeString(types.ToString(args[0])), nil
}
functions["html_unescape"] = func(args ...interface{}) (interface{}, error) {
},
"html_unescape": func(args ...interface{}) (interface{}, error) {
return html.UnescapeString(types.ToString(args[0])), nil
}
},
// hashing
functions["md5"] = func(args ...interface{}) (interface{}, error) {
"md5": func(args ...interface{}) (interface{}, error) {
hash := md5.Sum([]byte(types.ToString(args[0])))
return hex.EncodeToString(hash[:]), nil
}
functions["sha256"] = func(args ...interface{}) (interface{}, error) {
},
"sha256": func(args ...interface{}) (interface{}, error) {
h := sha256.New()
_, err := h.Write([]byte(types.ToString(args[0])))
@ -143,9 +119,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
return nil, err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
functions["sha1"] = func(args ...interface{}) (interface{}, error) {
},
"sha1": func(args ...interface{}) (interface{}, error) {
h := sha1.New()
_, err := h.Write([]byte(types.ToString(args[0])))
@ -153,27 +128,23 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
return nil, err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
functions["mmh3"] = func(args ...interface{}) (interface{}, error) {
},
"mmh3": func(args ...interface{}) (interface{}, error) {
return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil
}
},
// search
functions["contains"] = func(args ...interface{}) (interface{}, error) {
"contains": func(args ...interface{}) (interface{}, error) {
return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil
}
functions["regex"] = func(args ...interface{}) (interface{}, error) {
},
"regex": func(args ...interface{}) (interface{}, error) {
compiled, err := regexp.Compile(types.ToString(args[0]))
if err != nil {
return nil, err
}
return compiled.MatchString(types.ToString(args[1])), nil
}
},
// random generators
functions["rand_char"] = func(args ...interface{}) (interface{}, error) {
"rand_char": func(args ...interface{}) (interface{}, error) {
chars := letters + numbers
bad := ""
if len(args) >= 1 {
@ -184,9 +155,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
}
chars = trimAll(chars, bad)
return chars[rand.Intn(len(chars))], nil
}
functions["rand_base"] = func(args ...interface{}) (interface{}, error) {
},
"rand_base": func(args ...interface{}) (interface{}, error) {
l := 0
bad := ""
base := letters + numbers
@ -202,9 +172,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
}
base = trimAll(base, bad)
return randSeq(base, l), nil
}
functions["rand_text_alphanumeric"] = func(args ...interface{}) (interface{}, error) {
},
"rand_text_alphanumeric": func(args ...interface{}) (interface{}, error) {
l := 0
bad := ""
chars := letters + numbers
@ -217,9 +186,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
}
chars = trimAll(chars, bad)
return randSeq(chars, l), nil
}
functions["rand_text_alpha"] = func(args ...interface{}) (interface{}, error) {
},
"rand_text_alpha": func(args ...interface{}) (interface{}, error) {
l := 0
bad := ""
chars := letters
@ -232,9 +200,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
}
chars = trimAll(chars, bad)
return randSeq(chars, l), nil
}
functions["rand_text_numeric"] = func(args ...interface{}) (interface{}, error) {
},
"rand_text_numeric": func(args ...interface{}) (interface{}, error) {
l := 0
bad := ""
chars := numbers
@ -247,9 +214,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
}
chars = trimAll(chars, bad)
return randSeq(chars, l), nil
}
functions["rand_int"] = func(args ...interface{}) (interface{}, error) {
},
"rand_int": func(args ...interface{}) (interface{}, error) {
min := 0
max := math.MaxInt32
@ -260,17 +226,15 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
max = args[1].(int)
}
return rand.Intn(max-min) + min, nil
}
},
// Time Functions
functions["waitfor"] = func(args ...interface{}) (interface{}, error) {
"waitfor": func(args ...interface{}) (interface{}, error) {
seconds := args[0].(float64)
time.Sleep(time.Duration(seconds) * time.Second)
return true, nil
}
},
// deserialization Functions
functions["generate_java_gadget"] = func(args ...interface{}) (interface{}, error) {
"generate_java_gadget": func(args ...interface{}) (interface{}, error) {
gadget := args[0].(string)
cmd := args[1].(string)
@ -280,10 +244,23 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
}
data := deserialization.GenerateJavaGadget(gadget, cmd, encoding)
return data, nil
}
},
}
// HelperFunctions returns the dsl helper functions
func HelperFunctions() map[string]govaluate.ExpressionFunction {
return functions
}
// AddHelperFunction allows creation of additiona helper functions to be supported with templates
func AddHelperFunction(key string, value func(args ...interface{}) (interface{}, error)) error {
if _, ok := functions[key]; !ok {
functions[key] = value
return nil
}
return errors.New("duplicate helper function key defined")
}
func reverseString(s string) string {
runes := []rune(s)
for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {

2
v2/pkg/protocols/dns/dns.go
View File

@ -118,6 +118,8 @@ func questionTypeToInt(questionType string) uint16 {
question = dns.TypeMX
case "TXT":
question = dns.TypeTXT
case "DS":
question = dns.TypeDS
case "AAAA":
question = dns.TypeAAAA
}

27
v2/pkg/protocols/http/request.go
View File

@ -263,19 +263,6 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
err error
)
// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
if !request.original.Race {
dumpedRequest, err = dump(request, reqURL)
if err != nil {
return err
}
if r.options.Options.Debug || r.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL)
gologger.Print().Msgf("%s", string(dumpedRequest))
}
}
var formedURL string
var hostname string
timeStart := time.Now()
@ -314,6 +301,20 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
resp, err = r.httpClient.Do(request.request)
}
}
// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
if !request.original.Race {
dumpedRequest, err = dump(request, reqURL)
if err != nil {
return err
}
if r.options.Options.Debug || r.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL)
gologger.Print().Msgf("%s", string(dumpedRequest))
}
}
if resp == nil {
err = errors.New("no response got for request")
}

8
v2/pkg/templates/compile.go
View File

@ -18,7 +18,7 @@ import (
// Parse parses a yaml request template file
//nolint:gocritic // this cannot be passed by pointer
func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error) {
func Parse(filePath string, preprocessor Preprocessor, options protocols.ExecuterOptions) (*Template, error) {
template := &Template{}
f, err := os.Open(filePath)
@ -33,6 +33,10 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error
}
data = template.expandPreprocessors(data)
if preprocessor != nil {
data = preprocessor.Process(data)
}
err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
if err != nil {
return nil, err
@ -63,7 +67,7 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error
if err != nil {
return nil, errors.Wrap(err, "could not create workflow loader")
}
compileWorkflow(&options, compiled, loader)
compileWorkflow(preprocessor, &options, compiled, loader)
template.CompiledWorkflow = compiled
template.CompiledWorkflow.Options = &options
}

4
v2/pkg/templates/preprocessors.go
View File

@ -8,6 +8,10 @@ import (
"github.com/segmentio/ksuid"
)
type Preprocessor interface {
Process(data []byte) []byte
}
var preprocessorRegex = regexp.MustCompile(`\{\{([a-z0-9_]+)\}\}`)
// expandPreprocessors expands the pre-processors if any for a template data.

16
v2/pkg/templates/workflows.go
View File

@ -8,9 +8,9 @@ import (
)
// compileWorkflow compiles the workflow for execution
func compileWorkflow(options *protocols.ExecuterOptions, workflow *workflows.Workflow, loader compile.WorkflowLoader) {
func compileWorkflow(preprocessor Preprocessor, options *protocols.ExecuterOptions, workflow *workflows.Workflow, loader compile.WorkflowLoader) {
for _, workflow := range workflow.Workflows {
if err := parseWorkflow(workflow, options, loader); err != nil {
if err := parseWorkflow(preprocessor, workflow, options, loader); err != nil {
gologger.Warning().Msgf("Could not parse workflow: %v\n", err)
continue
}
@ -18,24 +18,24 @@ func compileWorkflow(options *protocols.ExecuterOptions, workflow *workflows.Wor
}
// parseWorkflow parses and compiles all templates in a workflow recursively
func parseWorkflow(workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions, loader compile.WorkflowLoader) error {
func parseWorkflow(preprocessor Preprocessor, workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions, loader compile.WorkflowLoader) error {
shouldNotValidate := false
if len(workflow.Subtemplates) > 0 || len(workflow.Matchers) > 0 {
shouldNotValidate = true
}
if err := parseWorkflowTemplate(workflow, options, loader, shouldNotValidate); err != nil {
if err := parseWorkflowTemplate(workflow, preprocessor, options, loader, shouldNotValidate); err != nil {
return err
}
for _, subtemplates := range workflow.Subtemplates {
if err := parseWorkflow(subtemplates, options, loader); err != nil {
if err := parseWorkflow(preprocessor, subtemplates, options, loader); err != nil {
gologger.Warning().Msgf("Could not parse workflow: %v\n", err)
continue
}
}
for _, matcher := range workflow.Matchers {
for _, subtemplates := range matcher.Subtemplates {
if err := parseWorkflow(subtemplates, options, loader); err != nil {
if err := parseWorkflow(preprocessor, subtemplates, options, loader); err != nil {
gologger.Warning().Msgf("Could not parse workflow: %v\n", err)
continue
}
@ -45,7 +45,7 @@ func parseWorkflow(workflow *workflows.WorkflowTemplate, options *protocols.Exec
}
// parseWorkflowTemplate parses a workflow template creating an executer
func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions, loader compile.WorkflowLoader, noValidate bool) error {
func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Preprocessor, options *protocols.ExecuterOptions, loader compile.WorkflowLoader, noValidate bool) error {
var paths []string
if len(workflow.Tags) > 0 {
@ -68,7 +68,7 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, options *protoc
Interactsh: options.Interactsh,
ProjectFile: options.ProjectFile,
}
template, err := Parse(path, opts)
template, err := Parse(path, preprocessor, opts)
if err != nil {
gologger.Warning().Msgf("Could not parse workflow template %s: %v\n", path, err)
continue