daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1296 from projectdiscovery/dsl_signatures 

feat: Improve DSL function UX
dev
Sandeep Singh 2021-12-09 12:48:18 +05:30 committed by GitHub
parent 22ce4e65bc 2d5784d992
commit 17ad60d528
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 583 additions and 389 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

746
v2/pkg/operators/common/dsl/dsl.go
View File

@ -15,10 +15,12 @@ import (
"math/rand"
"net/url"
"regexp"
"strconv"
"strings"
"time"
"github.com/Knetic/govaluate"
"github.com/logrusorgru/aurora"
"github.com/spaolacci/murmur3"
"github.com/projectdiscovery/gologger"
@ -27,374 +29,434 @@ import (
)
const (
numbers = "1234567890"
letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
withCutSetArgsSize = 2
withBaseRandArgsSize = 3
withMaxRandArgsSize = withCutSetArgsSize
numbers = "1234567890"
letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
)
var ErrDSLArguments = errors.New("invalid arguments provided to dsl")
var invalidDslFunctionError = errors.New("invalid DSL function signature")
var invalidDslFunctionMessageTemplate = "%w. correct method signature %q"
var functions = map[string]govaluate.ExpressionFunction{
"len": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
length := len(types.ToString(args[0]))
return float64(length), nil
},
"toupper": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
return strings.ToUpper(types.ToString(args[0])), nil
},
"tolower": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
return strings.ToLower(types.ToString(args[0])), nil
},
"replace": func(args ...interface{}) (interface{}, error) {
if len(args) != 3 {
return nil, ErrDSLArguments
}
return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil
},
"replace_regex": func(args ...interface{}) (interface{}, error) {
if len(args) != 3 {
return nil, ErrDSLArguments
}
compiled, err := regexp.Compile(types.ToString(args[1]))
if err != nil {
return nil, err
}
return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil
},
"trim": func(args ...interface{}) (interface{}, error) {
if len(args) != 2 {
return nil, ErrDSLArguments
}
return strings.Trim(types.ToString(args[0]), types.ToString(args[1])), nil
},
"trimleft": func(args ...interface{}) (interface{}, error) {
if len(args) != 2 {
return nil, ErrDSLArguments
}
return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil
},
"trimright": func(args ...interface{}) (interface{}, error) {
if len(args) != 2 {
return nil, ErrDSLArguments
}
return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil
},
"trimspace": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
return strings.TrimSpace(types.ToString(args[0])), nil
},
"trimprefix": func(args ...interface{}) (interface{}, error) {
if len(args) != 2 {
return nil, ErrDSLArguments
}
return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil
},
"trimsuffix": func(args ...interface{}) (interface{}, error) {
if len(args) != 2 {
return nil, ErrDSLArguments
}
return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil
},
"reverse": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
return reverseString(types.ToString(args[0])), nil
},
// encoding
"base64": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
var dslFunctions map[string]dslFunction
return sEnc, nil
},
"gzip": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
buffer := &bytes.Buffer{}
writer := gzip.NewWriter(buffer)
if _, err := writer.Write([]byte(args[0].(string))); err != nil {
return "", err
}
_ = writer.Close()
type dslFunction struct {
signature string
expressFunc govaluate.ExpressionFunction
}
return buffer.String(), nil
},
// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
"base64_py": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
return deserialization.InsertInto(sEnc, 76, '\n'), nil
},
"base64_decode": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
return base64.StdEncoding.DecodeString(types.ToString(args[0]))
},
"url_encode": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
return url.QueryEscape(types.ToString(args[0])), nil
},
"url_decode": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
return url.QueryUnescape(types.ToString(args[0]))
},
"hex_encode": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
return hex.EncodeToString([]byte(types.ToString(args[0]))), nil
},
"hex_decode": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
hx, _ := hex.DecodeString(types.ToString(args[0]))
return string(hx), nil
},
"html_escape": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
return html.EscapeString(types.ToString(args[0])), nil
},
"html_unescape": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
return html.UnescapeString(types.ToString(args[0])), nil
},
// hashing
"md5": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
hash := md5.Sum([]byte(types.ToString(args[0])))
func init() {
tempDslFunctions := map[string]func(string) dslFunction{
"len": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
length := len(types.ToString(args[0]))
return float64(length), nil
}),
"to_upper": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
return strings.ToUpper(types.ToString(args[0])), nil
}),
"to_lower": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
return strings.ToLower(types.ToString(args[0])), nil
}),
"replace": makeDslFunction(3, func(args ...interface{}) (interface{}, error) {
return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil
}),
"replace_regex": makeDslFunction(3, func(args ...interface{}) (interface{}, error) {
compiled, err := regexp.Compile(types.ToString(args[1]))
if err != nil {
return nil, err
}
return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil
}),
"trim": makeDslFunction(2, func(args ...interface{}) (interface{}, error) {
return strings.Trim(types.ToString(args[0]), types.ToString(args[1])), nil
}),
"trim_left": makeDslFunction(2, func(args ...interface{}) (interface{}, error) {
return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil
}),
"trim_right": makeDslFunction(2, func(args ...interface{}) (interface{}, error) {
return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil
}),
"trim_space": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
return strings.TrimSpace(types.ToString(args[0])), nil
}),
"trim_prefix": makeDslFunction(2, func(args ...interface{}) (interface{}, error) {
return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil
}),
"trim_suffix": makeDslFunction(2, func(args ...interface{}) (interface{}, error) {
return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil
}),
"reverse": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
return reverseString(types.ToString(args[0])), nil
}),
"base64": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
return base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))), nil
}),
"gzip": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
buffer := &bytes.Buffer{}
writer := gzip.NewWriter(buffer)
if _, err := writer.Write([]byte(args[0].(string))); err != nil {
return "", err
}
_ = writer.Close()
return hex.EncodeToString(hash[:]), nil
},
"sha256": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
h := sha256.New()
if _, err := h.Write([]byte(types.ToString(args[0]))); err != nil {
return nil, err
}
return hex.EncodeToString(h.Sum(nil)), nil
},
"sha1": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
h := sha1.New()
if _, err := h.Write([]byte(types.ToString(args[0]))); err != nil {
return nil, err
}
return hex.EncodeToString(h.Sum(nil)), nil
},
"mmh3": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil
},
// search
"contains": func(args ...interface{}) (interface{}, error) {
if len(args) != 2 {
return nil, ErrDSLArguments
}
return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil
},
"regex": func(args ...interface{}) (interface{}, error) {
if len(args) != 2 {
return nil, ErrDSLArguments
}
compiled, err := regexp.Compile(types.ToString(args[0]))
if err != nil {
return nil, err
}
return compiled.MatchString(types.ToString(args[1])), nil
},
// random generators
"rand_char": func(args ...interface{}) (interface{}, error) {
if len(args) != 2 {
return nil, ErrDSLArguments
}
chars := letters + numbers
bad := ""
if len(args) >= 1 {
chars = types.ToString(args[0])
}
if len(args) >= withCutSetArgsSize {
bad = types.ToString(args[1])
}
chars = trimAll(chars, bad)
return chars[rand.Intn(len(chars))], nil
},
"rand_base": func(args ...interface{}) (interface{}, error) {
if len(args) != 3 {
return nil, ErrDSLArguments
}
l := 0
bad := ""
base := letters + numbers
return buffer.String(), nil
}),
"base64_py": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
stdBase64 := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
return deserialization.InsertInto(stdBase64, 76, '\n'), nil
}),
"base64_decode": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
return base64.StdEncoding.DecodeString(types.ToString(args[0]))
}),
"url_encode": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
return url.QueryEscape(types.ToString(args[0])), nil
}),
"url_decode": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
return url.QueryUnescape(types.ToString(args[0]))
}),
"hex_encode": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
return hex.EncodeToString([]byte(types.ToString(args[0]))), nil
}),
"hex_decode": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
decodeString, err := hex.DecodeString(types.ToString(args[0]))
return decodeString, err
}),
"html_escape": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
return html.EscapeString(types.ToString(args[0])), nil
}),
"html_unescape": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
return html.UnescapeString(types.ToString(args[0])), nil
}),
"md5": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
hash := md5.Sum([]byte(types.ToString(args[0])))
return hex.EncodeToString(hash[:]), nil
}),
"sha256": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
hash := sha256.New()
if _, err := hash.Write([]byte(types.ToString(args[0]))); err != nil {
return nil, err
}
return hex.EncodeToString(hash.Sum(nil)), nil
}),
"sha1": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
hash := sha1.New()
if _, err := hash.Write([]byte(types.ToString(args[0]))); err != nil {
return nil, err
}
return hex.EncodeToString(hash.Sum(nil)), nil
}),
"mmh3": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil
}),
"contains": makeDslFunction(2, func(args ...interface{}) (interface{}, error) {
return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil
}),
"regex": makeDslFunction(2, func(args ...interface{}) (interface{}, error) {
compiled, err := regexp.Compile(types.ToString(args[0]))
if err != nil {
return nil, err
}
return compiled.MatchString(types.ToString(args[1])), nil
}),
"remove_bad_chars": makeDslFunction(2, func(args ...interface{}) (interface{}, error) {
input := types.ToString(args[0])
badChars := types.ToString(args[1])
return trimAll(input, badChars), nil
}),
"rand_char": makeDslWithOptionalArgsFunction(
"(optionalCharSet string) string",
func(args ...interface{}) (interface{}, error) {
charSet := letters + numbers
if len(args) >= 1 {
l = int(args[0].(float64))
}
if len(args) >= withCutSetArgsSize {
bad = types.ToString(args[1])
}
if len(args) >= withBaseRandArgsSize {
base = types.ToString(args[2])
}
base = trimAll(base, bad)
return randSeq(base, l), nil
},
"rand_text_alphanumeric": func(args ...interface{}) (interface{}, error) {
if len(args) != 2 {
return nil, ErrDSLArguments
}
l := 0
bad := ""
chars := letters + numbers
argSize := len(args)
if argSize != 0 && argSize != 1 {
return nil, invalidDslFunctionError
}
if len(args) >= 1 {
l = int(args[0].(float64))
}
if len(args) >= withCutSetArgsSize {
bad = types.ToString(args[1])
}
chars = trimAll(chars, bad)
return randSeq(chars, l), nil
},
"rand_text_alpha": func(args ...interface{}) (interface{}, error) {
if len(args) != 2 {
return nil, ErrDSLArguments
}
l := 0
bad := ""
chars := letters
if argSize >= 1 {
charSet = types.ToString(args[0])
}
if len(args) >= 1 {
l = int(args[0].(float64))
}
if len(args) >= withCutSetArgsSize {
bad = types.ToString(args[1])
}
chars = trimAll(chars, bad)
return randSeq(chars, l), nil
},
"rand_text_numeric": func(args ...interface{}) (interface{}, error) {
if len(args) != 2 {
return nil, ErrDSLArguments
}
l := 0
bad := ""
chars := numbers
return charSet[rand.Intn(len(charSet))], nil
},
),
"rand_base": makeDslWithOptionalArgsFunction(
"(length uint, optionalCharSet string) string",
func(args ...interface{}) (interface{}, error) {
var length int
charSet := letters + numbers
if len(args) >= 1 {
l = int(args[0].(float64))
}
if len(args) >= withCutSetArgsSize {
bad = types.ToString(args[1])
}
chars = trimAll(chars, bad)
return randSeq(chars, l), nil
},
"rand_int": func(args ...interface{}) (interface{}, error) {
if len(args) != 2 {
return nil, ErrDSLArguments
}
min := 0
max := math.MaxInt32
argSize := len(args)
if argSize < 1 || argSize > 3 {
return nil, invalidDslFunctionError
}
if len(args) >= 1 {
min = int(args[0].(float64))
}
if len(args) >= withMaxRandArgsSize {
max = int(args[1].(float64))
}
return rand.Intn(max-min) + min, nil
},
"unixtime": func(args ...interface{}) (interface{}, error) {
seconds := 0
if len(args) >= 1 {
seconds = int(args[0].(float64))
}
now := time.Now()
offset := now.Add(time.Duration(seconds) * time.Second)
return float64(offset.Unix()), nil
},
// Time Functions
"waitfor": func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, ErrDSLArguments
}
seconds := args[0].(float64)
time.Sleep(time.Duration(seconds) * time.Second)
return true, nil
},
// deserialization Functions
"generate_java_gadget": func(args ...interface{}) (interface{}, error) {
if len(args) != 3 {
return nil, ErrDSLArguments
}
gadget := args[0].(string)
cmd := args[1].(string)
length = int(args[0].(float64))
var encoding string
if len(args) > 2 {
encoding = args[2].(string)
if argSize == 2 {
charSet = types.ToString(args[1])
}
return randSeq(charSet, length), nil
},
),
"rand_text_alphanumeric": makeDslWithOptionalArgsFunction(
"(length uint, optionalBadChars string) string",
func(args ...interface{}) (interface{}, error) {
length := 0
badChars := ""
argSize := len(args)
if argSize != 1 && argSize != 2 {
return nil, invalidDslFunctionError
}
length = int(args[0].(float64))
if argSize == 2 {
badChars = types.ToString(args[1])
}
chars := trimAll(letters+numbers, badChars)
return randSeq(chars, length), nil
},
),
"rand_text_alpha": makeDslWithOptionalArgsFunction(
"(length uint, optionalBadChars string) string",
func(args ...interface{}) (interface{}, error) {
var length int
badChars := ""
argSize := len(args)
if argSize != 1 && argSize != 2 {
return nil, invalidDslFunctionError
}
length = int(args[0].(float64))
if argSize == 2 {
badChars = types.ToString(args[1])
}
chars := trimAll(letters, badChars)
return randSeq(chars, length), nil
},
),
"rand_text_numeric": makeDslWithOptionalArgsFunction(
"(length uint, optionalBadNumbers string) string",
func(args ...interface{}) (interface{}, error) {
argSize := len(args)
if argSize != 1 && argSize != 2 {
return nil, invalidDslFunctionError
}
length := args[0].(int)
badNumbers := ""
if argSize == 2 {
badNumbers = types.ToString(args[1])
}
chars := trimAll(numbers, badNumbers)
return randSeq(chars, length), nil
},
),
"rand_int": makeDslWithOptionalArgsFunction(
"(optionalMin, optionalMax uint) int",
func(args ...interface{}) (interface{}, error) {
argSize := len(args)
if argSize >= 2 {
return nil, invalidDslFunctionError
}
min := 0
max := math.MaxInt32
if argSize >= 1 {
min = args[0].(int)
}
if argSize == 2 {
max = args[1].(int)
}
return rand.Intn(max-min) + min, nil
},
),
"generate_java_gadget": makeDslFunction(3, func(args ...interface{}) (interface{}, error) {
gadget := args[0].(string)
cmd := args[1].(string)
encoding := args[2].(string)
data := deserialization.GenerateJavaGadget(gadget, cmd, encoding)
return data, nil
}),
"unix_time": makeDslWithOptionalArgsFunction(
"(optionalSeconds uint) float64",
func(args ...interface{}) (interface{}, error) {
seconds := 0
argSize := len(args)
if argSize != 0 && argSize != 1 {
return nil, invalidDslFunctionError
} else if argSize == 1 {
seconds = int(args[0].(uint))
}
offset := time.Now().Add(time.Duration(seconds) * time.Second)
return float64(offset.Unix()), nil
},
),
"wait_for": makeDslWithOptionalArgsFunction(
"(seconds uint)",
func(args ...interface{}) (interface{}, error) {
if len(args) != 1 {
return nil, invalidDslFunctionError
}
seconds := args[0].(uint)
time.Sleep(time.Duration(seconds) * time.Second)
return true, nil
},
),
"print_debug": makeDslWithOptionalArgsFunction(
"(args ...interface{})",
func(args ...interface{}) (interface{}, error) {
if len(args) < 1 {
return nil, invalidDslFunctionError
}
gologger.Info().Msgf("print_debug value: %s", fmt.Sprint(args))
return true, nil
},
),
}
dslFunctions = make(map[string]dslFunction, len(tempDslFunctions))
for funcName, dslFunc := range tempDslFunctions {
dslFunctions[funcName] = dslFunc(funcName)
}
}
func createSignaturePart(numberOfParameters int) string {
params := make([]string, 0, numberOfParameters)
for i := 1; i <= numberOfParameters; i++ {
params = append(params, "arg"+strconv.Itoa(i))
}
return fmt.Sprintf("(%s interface{}) interface{}", strings.Join(params, ", "))
}
func makeDslWithOptionalArgsFunction(signaturePart string, dslFunctionLogic govaluate.ExpressionFunction) func(functionName string) dslFunction {
return func(functionName string) dslFunction {
return dslFunction{
functionName + signaturePart,
dslFunctionLogic,
}
data := deserialization.GenerateJavaGadget(gadget, cmd, encoding)
return data, nil
},
// for debug purposes
"print_debug": func(args ...interface{}) (interface{}, error) {
gologger.Info().Msgf("print_debug value: %s", fmt.Sprint(args))
return true, nil
},
}
}
func makeDslFunction(numberOfParameters int, dslFunctionLogic govaluate.ExpressionFunction) func(functionName string) dslFunction {
return func(functionName string) dslFunction {
signature := functionName + createSignaturePart(numberOfParameters)
return dslFunction{
signature,
func(args ...interface{}) (interface{}, error) {
if len(args) != numberOfParameters {
return nil, fmt.Errorf(invalidDslFunctionMessageTemplate, invalidDslFunctionError, signature)
}
return dslFunctionLogic(args...)
},
}
}
}
// HelperFunctions returns the dsl helper functions
func HelperFunctions() map[string]govaluate.ExpressionFunction {
return functions
helperFunctions := make(map[string]govaluate.ExpressionFunction, len(dslFunctions))
for functionName, dslFunction := range dslFunctions {
helperFunctions[functionName] = dslFunction.expressFunc
helperFunctions[strings.ReplaceAll(functionName, "_", "")] = dslFunction.expressFunc // for backwards compatibility
}
return helperFunctions
}
// AddHelperFunction allows creation of additional helper functions to be supported with templates
//goland:noinspection GoUnusedExportedFunction
func AddHelperFunction(key string, value func(args ...interface{}) (interface{}, error)) error {
if _, ok := functions[key]; !ok {
functions[key] = value
if _, ok := dslFunctions[key]; !ok {
dslFunction := dslFunctions[key]
dslFunction.signature = "(args ...interface{}) interface{}"
dslFunction.expressFunc = value
return nil
}
return errors.New("duplicate helper function key defined")
}
func GetPrintableDslFunctionSignatures(noColor bool) string {
aggregateSignatures := func(values []string) string {
builder := &strings.Builder{}
for _, value := range values {
builder.WriteRune('\t')
builder.WriteString(value)
builder.WriteRune('\n')
}
return builder.String()
}
if noColor {
return aggregateSignatures(getDslFunctionSignatures())
}
return aggregateSignatures(colorizeDslFunctionSignatures())
}
func getDslFunctionSignatures() []string {
result := make([]string, 0, len(dslFunctions))
for _, dslFunction := range dslFunctions {
result = append(result, dslFunction.signature)
}
return result
}
var functionSignaturePattern = regexp.MustCompile(`(\w+)\s*\((?:([\w\d,\s]+)\s+([.\w\d{}&*]+))?\)([\s.\w\d{}&*]+)?`)
func colorizeDslFunctionSignatures() []string {
signatures := getDslFunctionSignatures()
colorToOrange := func(value string) string {
return aurora.Index(208, value).String()
}
result := make([]string, 0, len(signatures))
for _, signature := range signatures {
subMatchSlices := functionSignaturePattern.FindAllStringSubmatch(signature, -1)
if len(subMatchSlices) != 1 {
// TODO log when #1166 is implemented
return signatures
}
matches := subMatchSlices[0]
if len(matches) != 5 {
// TODO log when #1166 is implemented
return signatures
}
functionParameters := strings.Split(matches[2], ",")
var coloredParameterAndTypes []string
for _, functionParameter := range functionParameters {
functionParameter = strings.TrimSpace(functionParameter)
paramAndType := strings.Split(functionParameter, " ")
if len(paramAndType) == 1 {
coloredParameterAndTypes = append(coloredParameterAndTypes, paramAndType[0])
} else if len(paramAndType) == 2 {
coloredParameterAndTypes = append(coloredParameterAndTypes, fmt.Sprintf("%s %s", paramAndType[0], colorToOrange(paramAndType[1])))
}
}
highlightedParams := strings.TrimSpace(fmt.Sprintf("%s %s", strings.Join(coloredParameterAndTypes, ", "), colorToOrange(matches[3])))
colorizedDslSignature := fmt.Sprintf("%s(%s)%s", aurora.BrightYellow(matches[1]).String(), highlightedParams, colorToOrange(matches[4]))
result = append(result, colorizedDslSignature)
}
return result
}
func reverseString(s string) string {
runes := []rune(s)
for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {

90
v2/pkg/operators/common/dsl/dsl_test.go
View File

@ -2,12 +2,14 @@ package dsl
import (
"compress/gzip"
"fmt"
"io/ioutil"
"strings"
"testing"
"time"
"github.com/Knetic/govaluate"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
@ -46,3 +48,91 @@ func TestDSLGzipSerialize(t *testing.T) {
require.Equal(t, "hello world", string(data), "could not get gzip encoded data")
}
func Test1(t *testing.T) {
type testCase struct {
methodName string
arguments []interface{}
expected interface{}
err string
}
toUpperSignatureError := createSignatureError("to_upper(arg1 interface{}) interface{}")
removeBadCharsSignatureError := createSignatureError("remove_bad_chars(arg1, arg2 interface{}) interface{}")
testCases := []testCase{
{"to_upper", []interface{}{}, nil, toUpperSignatureError},
{"to_upper", []interface{}{"a"}, "A", ""},
{"toupper", []interface{}{"a"}, "A", ""},
{"to_upper", []interface{}{"a", "b", "c"}, nil, toUpperSignatureError},
{"remove_bad_chars", []interface{}{}, nil, removeBadCharsSignatureError},
{"remove_bad_chars", []interface{}{"a"}, nil, removeBadCharsSignatureError},
{"remove_bad_chars", []interface{}{"abba baab", "b"}, "aa aa", ""},
{"remove_bad_chars", []interface{}{"a", "b", "c"}, nil, removeBadCharsSignatureError},
}
helperFunctions := HelperFunctions()
for _, currentTestCase := range testCases {
methodName := currentTestCase.methodName
t.Run(methodName, func(t *testing.T) {
actualResult, err := helperFunctions[methodName](currentTestCase.arguments...)
if currentTestCase.err == "" {
assert.Nil(t, err)
} else {
assert.Equal(t, err.Error(), currentTestCase.err)
}
assert.Equal(t, currentTestCase.expected, actualResult)
})
}
}
func createSignatureError(signature string) string {
return fmt.Errorf(invalidDslFunctionMessageTemplate, invalidDslFunctionError, signature).Error()
}
func Test(t *testing.T) {
expectedColorizedSignatures := []string{
"\x1b[93mbase64_py\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mprint_debug\x1b[0m(args \x1b[38;5;208m...interface{}\x1b[0m)\x1b[38;5;208m\x1b[0m",
"\x1b[93mregex\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mmmh3\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mto_lower\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mmd5\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mreplace_regex\x1b[0m(arg1, arg2, arg3 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mhtml_unescape\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mhex_encode\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mrand_base\x1b[0m(length \x1b[38;5;208muint\x1b[0m, optionalCharSet \x1b[38;5;208mstring\x1b[0m)\x1b[38;5;208m string\x1b[0m",
"\x1b[93msha1\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mtrim_right\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mwait_for\x1b[0m(seconds \x1b[38;5;208muint\x1b[0m)\x1b[38;5;208m\x1b[0m",
"\x1b[93mtrim\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93murl_encode\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mto_upper\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mrand_text_alpha\x1b[0m(length \x1b[38;5;208muint\x1b[0m, optionalBadChars \x1b[38;5;208mstring\x1b[0m)\x1b[38;5;208m string\x1b[0m",
"\x1b[93msha256\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mgzip\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mlen\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mtrim_space\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mrand_int\x1b[0m(optionalMin, optionalMax \x1b[38;5;208muint\x1b[0m)\x1b[38;5;208m int\x1b[0m",
"\x1b[93mremove_bad_chars\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mrand_char\x1b[0m(optionalCharSet \x1b[38;5;208mstring\x1b[0m)\x1b[38;5;208m string\x1b[0m",
"\x1b[93mreverse\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mhtml_escape\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mbase64\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mbase64_decode\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mhex_decode\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mtrim_prefix\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93murl_decode\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mreplace\x1b[0m(arg1, arg2, arg3 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mtrim_suffix\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mrand_text_numeric\x1b[0m(length \x1b[38;5;208muint\x1b[0m, optionalBadNumbers \x1b[38;5;208mstring\x1b[0m)\x1b[38;5;208m string\x1b[0m",
"\x1b[93mcontains\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mgenerate_java_gadget\x1b[0m(arg1, arg2, arg3 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93munix_time\x1b[0m(optionalSeconds \x1b[38;5;208muint\x1b[0m)\x1b[38;5;208m float64\x1b[0m",
"\x1b[93mtrim_left\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m",
"\x1b[93mrand_text_alphanumeric\x1b[0m(length \x1b[38;5;208muint\x1b[0m, optionalBadChars \x1b[38;5;208mstring\x1b[0m)\x1b[38;5;208m string\x1b[0m",
}
assert.ElementsMatch(t, expectedColorizedSignatures, colorizeDslFunctionSignatures())
}

21
v2/pkg/operators/matchers/compile.go
View File

@ -55,12 +55,12 @@ func (matcher *Matcher) CompileMatchers() error {
}
// Compile the dsl expressions
for _, expr := range matcher.DSL {
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
for _, dslExpression := range matcher.DSL {
compiledExpression, err := govaluate.NewEvaluableExpressionWithFunctions(dslExpression, dsl.HelperFunctions())
if err != nil {
return fmt.Errorf("could not compile dsl: %s", expr)
return &DslCompilationError{DslSignature: dslExpression, WrappedError: err}
}
matcher.dslCompiled = append(matcher.dslCompiled, compiled)
matcher.dslCompiled = append(matcher.dslCompiled, compiledExpression)
}
// Set up the condition type, if any.
@ -83,3 +83,16 @@ func (matcher *Matcher) CompileMatchers() error {
}
return nil
}
type DslCompilationError struct {
DslSignature string
WrappedError error
}
func (e *DslCompilationError) Error() string {
return fmt.Sprintf("could not compile DSL expression: %s. %v", e.DslSignature, e.WrappedError)
}
func (e *DslCompilationError) Unwrap() error {
return e.WrappedError
}

49
v2/pkg/operators/matchers/match.go
View File

@ -4,6 +4,7 @@ import (
"strings"
"github.com/Knetic/govaluate"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
@ -57,17 +58,19 @@ func (matcher *Matcher) MatchWords(corpus string, data map[string]interface{}) (
var err error
word, err = expressions.Evaluate(word, data)
if err != nil {
gologger.Warning().Msgf("Error while evaluating word matcher: %q", word)
continue
}
// Continue if the word doesn't match
if !strings.Contains(corpus, word) {
// If we are in an AND request and a match failed,
// return false as the AND condition fails on any single mismatch.
if matcher.condition == ANDCondition {
switch matcher.condition {
case ANDCondition:
return false, []string{}
case ORCondition:
continue
}
// Continue with the flow since it's an OR Condition.
continue
}
// If the condition was an OR, return on the first match.
@ -94,11 +97,12 @@ func (matcher *Matcher) MatchRegex(corpus string) (bool, []string) {
if !regex.MatchString(corpus) {
// If we are in an AND request and a match failed,
// return false as the AND condition fails on any single mismatch.
if matcher.condition == ANDCondition {
switch matcher.condition {
case ANDCondition:
return false, []string{}
case ORCondition:
continue
}
// Continue with the flow since it's an OR Condition.
continue
}
currentMatches := regex.FindAllString(corpus, -1)
@ -125,11 +129,12 @@ func (matcher *Matcher) MatchBinary(corpus string) (bool, []string) {
if !strings.Contains(corpus, binary) {
// If we are in an AND request and a match failed,
// return false as the AND condition fails on any single mismatch.
if matcher.condition == ANDCondition {
switch matcher.condition {
case ANDCondition:
return false, []string{}
case ORCondition:
continue
}
// Continue with the flow since it's an OR Condition.
continue
}
// If the condition was an OR, return on the first match.
@ -149,37 +154,43 @@ func (matcher *Matcher) MatchBinary(corpus string) (bool, []string) {
// MatchDSL matches on a generic map result
func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool {
logExpressionEvaluationFailure := func (matcherName string, err error) {
gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcherName, err.Error())
}
// Iterate over all the expressions accepted as valid
for i, expression := range matcher.dslCompiled {
if varErr := expressions.ContainsUnresolvedVariables(expression.String()); varErr != nil {
resolvedExpression, err := expressions.Evaluate(expression.String(), data)
if err != nil {
gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcher.Name, err.Error())
logExpressionEvaluationFailure(matcher.Name, err)
return false
}
expression, err = govaluate.NewEvaluableExpressionWithFunctions(resolvedExpression, dsl.HelperFunctions())
if err != nil {
gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcher.Name, err.Error())
logExpressionEvaluationFailure(matcher.Name, err)
return false
}
}
result, err := expression.Evaluate(data)
if err != nil {
gologger.Warning().Msgf(err.Error())
continue
}
var bResult bool
bResult, ok := result.(bool)
// Continue if the regex doesn't match
if !ok || !bResult {
if boolResult, ok := result.(bool); !ok {
gologger.Warning().Msgf("The return value of a DSL statement must return a boolean value.")
continue
} else if !boolResult {
// If we are in an AND request and a match failed,
// return false as the AND condition fails on any single mismatch.
if matcher.condition == ANDCondition {
switch matcher.condition {
case ANDCondition:
return false
case ORCondition:
continue
}
// Continue with the flow since it's an OR Condition.
continue
}
// If the condition was an OR, return on the first match.

17
v2/pkg/protocols/common/executer/executer.go
View File

@ -1,9 +1,14 @@
package executer
import (
"fmt"
"strings"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer"
@ -24,8 +29,20 @@ func NewExecuter(requests []protocols.Request, options *protocols.ExecuterOption
// Compile compiles the execution generators preparing any requests possible.
func (e *Executer) Compile() error {
cliOptions := e.options.Options
for _, request := range e.requests {
if err := request.Compile(e.options); err != nil {
var dslCompilationError *matchers.DslCompilationError
if errors.As(err, &dslCompilationError) {
if cliOptions.Verbose {
rawErrorMessage := dslCompilationError.Error()
formattedErrorMessage := strings.ToUpper(rawErrorMessage[:1]) + rawErrorMessage[1:] + "."
gologger.Warning().Msgf(formattedErrorMessage)
gologger.Info().Msgf("The available custom DSL functions are:")
fmt.Println(dsl.GetPrintableDslFunctionSignatures(cliOptions.NoColor))
}
}
return err
}
}

12
v2/pkg/reporting/exporters/es/elasticsearch.go
View File

@ -82,14 +82,14 @@ func New(option *Options) (*Exporter, error) {
}
// Export exports a passed result event to elasticsearch
func (i *Exporter) Export(event *output.ResultEvent) error {
func (exporter *Exporter) Export(event *output.ResultEvent) error {
// creating a request
req, err := http.NewRequest(http.MethodPost, i.url, nil)
req, err := http.NewRequest(http.MethodPost, exporter.url, nil)
if err != nil {
return errors.Wrap(err, "could not make request")
}
if len(i.authentication) > 0 {
req.Header.Add("Authorization", i.authentication)
if len(exporter.authentication) > 0 {
req.Header.Add("Authorization", exporter.authentication)
}
req.Header.Add("Content-Type", "application/json")
@ -103,7 +103,7 @@ func (i *Exporter) Export(event *output.ResultEvent) error {
}
req.Body = ioutil.NopCloser(bytes.NewReader(b))
res, err := i.elasticsearch.Do(req)
res, err := exporter.elasticsearch.Do(req)
if err != nil {
return err
}
@ -120,6 +120,6 @@ func (i *Exporter) Export(event *output.ResultEvent) error {
}
// Close closes the exporter after operation
func (i *Exporter) Close() error {
func (exporter *Exporter) Close() error {
return nil
}

6
v2/pkg/reporting/exporters/markdown/markdown.go
View File

@ -37,7 +37,7 @@ func New(options *Options) (*Exporter, error) {
}
// Export exports a passed result event to markdown
func (i *Exporter) Export(event *output.ResultEvent) error {
func (exporter *Exporter) Export(event *output.ResultEvent) error {
summary := format.Summary(event)
description := format.MarkdownDescription(event)
@ -66,11 +66,11 @@ func (i *Exporter) Export(event *output.ResultEvent) error {
dataBuilder.WriteString(description)
data := dataBuilder.Bytes()
return ioutil.WriteFile(filepath.Join(i.directory, finalFilename), data, 0644)
return ioutil.WriteFile(filepath.Join(exporter.directory, finalFilename), data, 0644)
}
// Close closes the exporter after operation
func (i *Exporter) Close() error {
func (exporter *Exporter) Close() error {
return nil
}

31
v2/pkg/reporting/exporters/sarif/sarif.go
View File

@ -51,8 +51,8 @@ func New(options *Options) (*Exporter, error) {
}
// Export exports a passed result event to sarif structure
func (i *Exporter) Export(event *output.ResultEvent) error {
templatePath := strings.TrimPrefix(event.TemplatePath, i.home)
func (exporter *Exporter) Export(event *output.ResultEvent) error {
templatePath := strings.TrimPrefix(event.TemplatePath, exporter.home)
h := sha1.New()
_, _ = h.Write([]byte(event.Host))
@ -67,7 +67,7 @@ func (i *Exporter) Export(event *output.ResultEvent) error {
}
var templateURL string
if strings.HasPrefix(event.TemplatePath, i.home) {
if strings.HasPrefix(event.TemplatePath, exporter.home) {
templateURL = "https://github.com/projectdiscovery/nuclei-templates/blob/master" + templatePath
} else {
templateURL = "https://github.com/projectdiscovery/nuclei-templates"
@ -78,19 +78,20 @@ func (i *Exporter) Export(event *output.ResultEvent) error {
ruleDescription = event.Info.Description
}
i.mutex.Lock()
defer i.mutex.Unlock()
exporter.mutex.Lock()
defer exporter.mutex.Unlock()
_ = i.run.AddRule(templateID).
_ = exporter.run.AddRule(templateID).
WithDescription(ruleName).
WithHelp(fullDescription).
WithHelpURI(templateURL).
WithFullDescription(sarif.NewMultiformatMessageString(ruleDescription))
result := i.run.AddResult(templateID).
result := exporter.run.AddResult(templateID).
WithMessage(sarif.NewMessage().WithText(event.Host)).
WithLevel(sarifSeverity)
// Also write file match metadata to file
// Also write file match metadata to file
if event.Type == "file" && (event.FileToIndexPosition != nil && len(event.FileToIndexPosition) > 0) {
for file, line := range event.FileToIndexPosition {
result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(ruleName)).WithPhysicalLocation(
@ -124,18 +125,18 @@ func getSarifSeverity(event *output.ResultEvent) string {
}
// Close closes the exporter after operation
func (i *Exporter) Close() error {
i.mutex.Lock()
defer i.mutex.Unlock()
func (exporter *Exporter) Close() error {
exporter.mutex.Lock()
defer exporter.mutex.Unlock()
i.sarif.AddRun(i.run)
if len(i.run.Results) == 0 {
exporter.sarif.AddRun(exporter.run)
if len(exporter.run.Results) == 0 {
return nil // do not write when no results
}
file, err := os.Create(i.options.File)
file, err := os.Create(exporter.options.File)
if err != nil {
return errors.Wrap(err, "could not create sarif output file")
}
defer file.Close()
return i.sarif.Write(file)
return exporter.sarif.Write(file)
}