Merge pull request #2055 from projectdiscovery/new_dsl_functions

DSL function changes
dev
Sandeep Singh 2022-06-10 14:33:54 +05:30 committed by GitHub
commit 48c95161e3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 279 additions and 178 deletions

View File

@ -16,57 +16,65 @@ requests:
04: {{base64_py("Hello")}}
05: {{concat("Hello", "world")}}
06: {{contains("Hello", "lo")}}
07: {{generate_java_gadget("commons-collections3.1", "wget http://{{interactsh-url}}", "base64")}}
08: {{gzip("Hello")}}
09: {{hex_decode("6161")}}
10: {{hex_encode("aa")}}
11: {{html_escape("<body>test</body>")}}
12: {{html_unescape("&lt;body&gt;test&lt;/body&gt;")}}
13: {{len("Hello")}}
14: {{len(5555)}}
15: {{md5("Hello")}}
16: {{md5(1234)}}
17: {{mmh3("Hello")}}
18: {{print_debug(1+2, "Hello")}}
19: {{rand_base(5, "abc")}}
20: {{rand_base(5, "")}}
21: {{rand_base(5)}}
22: {{rand_char("abc")}}
23: {{rand_char("")}}
24: {{rand_char()}}
25: {{rand_int(1, 10)}}
26: {{rand_int(10)}}
27: {{rand_int()}}
28: {{rand_ip("192.168.0.0/24")}}
29: {{rand_ip("2002:c0a8::/24")}}
30: {{rand_ip("192.168.0.0/24","10.0.100.0/24")}}
31: {{rand_text_alpha(10, "abc")}}
32: {{rand_text_alpha(10, "")}}
33: {{rand_text_alpha(10)}}
34: {{rand_text_alphanumeric(10, "ab12")}}
35: {{rand_text_alphanumeric(10)}}
36: {{rand_text_numeric(10, 123)}}
37: {{rand_text_numeric(10)}}
38: {{regex("H([a-z]+)o", "Hello")}}
39: {{remove_bad_chars("abcd", "bc")}}
40: {{repeat("a", 5)}}
41: {{replace("Hello", "He", "Ha")}}
42: {{replace_regex("He123llo", "(\\d+)", "")}}
43: {{reverse("abc")}}
44: {{sha1("Hello")}}
45: {{sha256("Hello")}}
46: {{to_lower("HELLO")}}
47: {{to_upper("hello")}}
48: {{trim("aaaHelloddd", "ad")}}
49: {{trim_left("aaaHelloddd", "ad")}}
50: {{trim_prefix("aaHelloaa", "aa")}}
51: {{trim_right("aaaHelloddd", "ad")}}
52: {{trim_space(" Hello ")}}
53: {{trim_suffix("aaHelloaa", "aa")}}
54: {{unix_time(10)}}
55: {{url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1")}}
56: {{url_encode("https://projectdiscovery.io/test?a=1")}}
57: {{wait_for(1)}}
07: {{date_time("%Y-%M-%D")}}
08: {{date_time("%Y-%M-%D", unix_time())}}
09: {{date_time("%H-%m")}}
10: {{date_time("02-01-2006 15:04")}}
11: {{date_time("02-01-2006 15:04", unix_time())}}
12: {{generate_java_gadget("commons-collections3.1", "wget http://{{interactsh-url}}", "base64")}}
13: {{gzip("Hello")}}
14: {{hex_decode("6161")}}
15: {{hex_encode("aa")}}
16: {{hmac("sha1", "test", "scrt")}}
17: {{hmac("sha256", "test", "scrt")}}
18: {{html_escape("<body>test</body>")}}
19: {{html_unescape("&lt;body&gt;test&lt;/body&gt;")}}
20: {{join("_", "hello", "world")}}
21: {{len("Hello")}}
22: {{len(5555)}}
23: {{md5("Hello")}}
24: {{md5(1234)}}
25: {{mmh3("Hello")}}
26: {{print_debug(1+2, "Hello")}}
27: {{rand_base(5, "abc")}}
28: {{rand_base(5, "")}}
29: {{rand_base(5)}}
30: {{rand_char("abc")}}
31: {{rand_char("")}}
32: {{rand_char()}}
33: {{rand_int(1, 10)}}
34: {{rand_int(10)}}
35: {{rand_int()}}
36: {{rand_ip("192.168.0.0/24")}}
37: {{rand_ip("2002:c0a8::/24")}}
38: {{rand_ip("192.168.0.0/24","10.0.100.0/24")}}
39: {{rand_text_alpha(10, "abc")}}
40: {{rand_text_alpha(10, "")}}
41: {{rand_text_alpha(10)}}
42: {{rand_text_alphanumeric(10, "ab12")}}
43: {{rand_text_alphanumeric(10)}}
44: {{rand_text_numeric(10, 123)}}
45: {{rand_text_numeric(10)}}
46: {{regex("H([a-z]+)o", "Hello")}}
47: {{remove_bad_chars("abcd", "bc")}}
48: {{repeat("a", 5)}}
49: {{replace("Hello", "He", "Ha")}}
50: {{replace_regex("He123llo", "(\\d+)", "")}}
51: {{reverse("abc")}}
52: {{sha1("Hello")}}
53: {{sha256("Hello")}}
54: {{to_lower("HELLO")}}
55: {{to_upper("hello")}}
56: {{trim("aaaHelloddd", "ad")}}
57: {{trim_left("aaaHelloddd", "ad")}}
58: {{trim_prefix("aaHelloaa", "aa")}}
59: {{trim_right("aaaHelloddd", "ad")}}
60: {{trim_space(" Hello ")}}
61: {{trim_suffix("aaHelloaa", "aa")}}
62: {{unix_time(10)}}
63: {{url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1")}}
64: {{url_encode("https://projectdiscovery.io/test?a=1")}}
65: {{wait_for(1)}}
extractors:
- type: regex

View File

@ -256,7 +256,7 @@ func (h *httpDSLFunctions) Execute(filePath string) error {
}
totalExtracted := strings.Split(submatch[1], ",")
numberOfDslFunctions := 57
numberOfDslFunctions := 65
if len(totalExtracted) != numberOfDslFunctions {
return errors.New("incorrect number of results")
}

View File

@ -4,12 +4,14 @@ import (
"bytes"
"compress/gzip"
"compress/zlib"
"crypto/hmac"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"hash"
"html"
"io"
"math"
@ -45,6 +47,7 @@ var invalidDslFunctionMessageTemplate = "%w. correct method signature %q"
var dslFunctions map[string]dslFunction
var functionSignaturePattern = regexp.MustCompile(`(\w+)\s*\((?:([\w\d,\s]+)\s+([.\w\d{}&*]+))?\)([\s.\w\d{}&*]+)?`)
var dateFormatRegex = regexp.MustCompile("%([A-Za-z])")
type dslFunction struct {
@ -153,58 +156,29 @@ func init() {
_ = reader.Close()
return string(data), nil
}),
"date": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
item := types.ToString(args[0])
submatches := dateFormatRegex.FindAllStringSubmatch(item, -1)
for _, value := range submatches {
if len(value) < 2 {
continue
"date_time": makeDslWithOptionalArgsFunction(
"(dateTimeFormat string, optionalUnixTime interface{}) string",
func(arguments ...interface{}) (interface{}, error) {
dateTimeFormat := types.ToString(arguments[0])
dateTimeFormatFragment := dateFormatRegex.FindAllStringSubmatch(dateTimeFormat, -1)
argumentsSize := len(arguments)
if argumentsSize < 1 && argumentsSize > 2 {
return nil, errors.New("invalid number of arguments")
}
now := time.Now()
switch value[1] {
case "Y", "y":
item = strings.ReplaceAll(item, value[0], appendSingleDigitZero(strconv.Itoa(now.Year())))
case "M", "m":
item = strings.ReplaceAll(item, value[0], appendSingleDigitZero(strconv.Itoa(int(now.Month()))))
case "D", "d":
item = strings.ReplaceAll(item, value[0], appendSingleDigitZero(strconv.Itoa(now.Day())))
default:
return nil, fmt.Errorf("invalid date format string: %s", value[0])
currentTime, err := getCurrentTimeFromUserInput(arguments)
if err != nil {
return nil, err
}
if len(dateTimeFormatFragment) > 0 {
return doSimpleTimeFormat(dateTimeFormatFragment, currentTime, dateTimeFormat)
} else {
return currentTime.Format(dateTimeFormat), nil
}
return item, nil
}),
"time": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
item := types.ToString(args[0])
submatches := dateFormatRegex.FindAllStringSubmatch(item, -1)
for _, value := range submatches {
if len(value) < 2 {
continue
}
now := time.Now()
switch value[1] {
case "H", "h":
item = strings.ReplaceAll(item, value[0], appendSingleDigitZero(strconv.Itoa(now.Hour())))
case "M", "m":
item = strings.ReplaceAll(item, value[0], appendSingleDigitZero(strconv.Itoa(now.Minute())))
case "S", "s":
item = strings.ReplaceAll(item, value[0], appendSingleDigitZero(strconv.Itoa(now.Second())))
default:
return nil, fmt.Errorf("invalid time format string: %s", value[0])
}
}
return item, nil
}),
"timetostring": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
if got, ok := args[0].(time.Time); ok {
return got.String(), nil
}
if got, ok := args[0].(float64); ok {
seconds, nanoseconds := math.Modf(got)
return time.Unix(int64(seconds), int64(nanoseconds)).String(), nil
}
return nil, fmt.Errorf("invalid time format: %T", args[0])
}),
},
),
"base64_py": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
stdBase64 := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
@ -227,6 +201,25 @@ func init() {
decodeString, err := hex.DecodeString(types.ToString(args[0]))
return string(decodeString), err
}),
"hmac": makeDslFunction(3, func(args ...interface{}) (interface{}, error) {
hashAlgorithm := args[0]
data := args[1].(string)
secretKey := args[2].(string)
var hashFunction func() hash.Hash
switch hashAlgorithm {
case "sha1", "sha-1":
hashFunction = sha1.New
case "sha256", "sha-256":
hashFunction = sha256.New
default:
return nil, fmt.Errorf("unsupported hash algorithm: '%s'", hashAlgorithm)
}
h := hmac.New(hashFunction, []byte(secretKey))
h.Write([]byte(data))
return hex.EncodeToString(h.Sum(nil)), nil
}),
"html_escape": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
return html.EscapeString(types.ToString(args[0])), nil
}),
@ -234,22 +227,13 @@ func init() {
return html.UnescapeString(types.ToString(args[0])), nil
}),
"md5": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
hash := md5.Sum([]byte(types.ToString(args[0])))
return hex.EncodeToString(hash[:]), nil
return toHexEncodedHash(md5.New(), types.ToString(args[0]))
}),
"sha256": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
hash := sha256.New()
if _, err := hash.Write([]byte(types.ToString(args[0]))); err != nil {
return nil, err
}
return hex.EncodeToString(hash.Sum(nil)), nil
return toHexEncodedHash(sha256.New(), types.ToString(args[0]))
}),
"sha1": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
hash := sha1.New()
if _, err := hash.Write([]byte(types.ToString(args[0]))); err != nil {
return nil, err
}
return hex.EncodeToString(hash.Sum(nil)), nil
return toHexEncodedHash(sha1.New(), types.ToString(args[0]))
}),
"mmh3": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
hasher := murmur3.New32WithSeed(0)
@ -269,6 +253,24 @@ func init() {
return builder.String(), nil
},
),
"join": makeDslWithOptionalArgsFunction(
"(separator string, elements ...interface{}) string",
func(arguments ...interface{}) (interface{}, error) {
argumentsSize := len(arguments)
if argumentsSize < 2 {
return nil, errors.New("incorrect number of arguments received")
}
separator := types.ToString(arguments[0])
elements := arguments[1:argumentsSize]
stringElements := make([]string, 0, argumentsSize)
for _, element := range elements {
stringElements = append(stringElements, types.ToString(element))
}
return strings.Join(stringElements, separator), nil
},
),
"regex": makeDslFunction(2, func(args ...interface{}) (interface{}, error) {
compiled, err := regexp.Compile(types.ToString(args[0]))
if err != nil {
@ -511,26 +513,6 @@ func init() {
}
}
// appendSingleDigitZero appends zero at front if not exists already doing two digit padding
func appendSingleDigitZero(value string) string {
if len(value) == 1 && (!strings.HasPrefix(value, "0") || value == "0") {
builder := &strings.Builder{}
builder.WriteRune('0')
builder.WriteString(value)
newVal := builder.String()
return newVal
}
return value
}
func createSignaturePart(numberOfParameters int) string {
params := make([]string, 0, numberOfParameters)
for i := 1; i <= numberOfParameters; i++ {
params = append(params, "arg"+strconv.Itoa(i))
}
return fmt.Sprintf("(%s interface{}) interface{}", strings.Join(params, ", "))
}
func makeDslWithOptionalArgsFunction(signaturePart string, dslFunctionLogic govaluate.ExpressionFunction) func(functionName string) dslFunction {
return func(functionName string) dslFunction {
return dslFunction{
@ -555,6 +537,14 @@ func makeDslFunction(numberOfParameters int, dslFunctionLogic govaluate.Expressi
}
}
func createSignaturePart(numberOfParameters int) string {
params := make([]string, 0, numberOfParameters)
for i := 1; i <= numberOfParameters; i++ {
params = append(params, "arg"+strconv.Itoa(i))
}
return fmt.Sprintf("(%s interface{}) interface{}", strings.Join(params, ", "))
}
// HelperFunctions returns the dsl helper functions
func HelperFunctions() map[string]govaluate.ExpressionFunction {
helperFunctions := make(map[string]govaluate.ExpressionFunction, len(dslFunctions))
@ -608,8 +598,6 @@ func getDslFunctionSignatures() []string {
return result
}
var functionSignaturePattern = regexp.MustCompile(`(\w+)\s*\((?:([\w\d,\s]+)\s+([.\w\d{}&*]+))?\)([\s.\w\d{}&*]+)?`)
func colorizeDslFunctionSignatures() []string {
signatures := getDslFunctionSignatures()
@ -675,3 +663,75 @@ func randSeq(base string, n int) string {
}
return string(b)
}
func toHexEncodedHash(hashToUse hash.Hash, data string) (interface{}, error) {
if _, err := hashToUse.Write([]byte(data)); err != nil {
return nil, err
}
return hex.EncodeToString(hashToUse.Sum(nil)), nil
}
func doSimpleTimeFormat(dateTimeFormatFragment [][]string, currentTime time.Time, dateTimeFormat string) (interface{}, error) {
for _, currentFragment := range dateTimeFormatFragment {
if len(currentFragment) < 2 {
continue
}
prefixedFormatFragment := currentFragment[0]
switch currentFragment[1] {
case "Y", "y":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Year())
case "M":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, int(currentTime.Month()))
case "D", "d":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Day())
case "H", "h":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Hour())
case "m":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Minute())
case "S", "s":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Second())
default:
return nil, fmt.Errorf("invalid date time format string: %s", prefixedFormatFragment)
}
}
return dateTimeFormat, nil
}
func getCurrentTimeFromUserInput(arguments []interface{}) (time.Time, error) {
var currentTime time.Time
if len(arguments) == 2 {
switch inputUnixTime := arguments[1].(type) {
case time.Time:
currentTime = inputUnixTime
case string:
unixTime, err := strconv.ParseInt(inputUnixTime, 10, 64)
if err != nil {
return time.Time{}, errors.New("invalid argument type")
}
currentTime = time.Unix(unixTime, 0)
case int64, float64:
currentTime = time.Unix(int64(inputUnixTime.(float64)), 0)
default:
return time.Time{}, errors.New("invalid argument type")
}
} else {
currentTime = time.Now()
}
return currentTime, nil
}
func formatDateTime(inputFormat string, matchValue string, timeFragment int) string {
return strings.ReplaceAll(inputFormat, matchValue, appendSingleDigitZero(strconv.Itoa(timeFragment)))
}
// appendSingleDigitZero appends zero at front if not exists already doing two digit padding
func appendSingleDigitZero(value string) string {
if len(value) == 1 && (!strings.HasPrefix(value, "0") || value == "0") {
builder := &strings.Builder{}
builder.WriteRune('0')
builder.WriteString(value)
newVal := builder.String()
return newVal
}
return value
}

View File

@ -51,16 +51,44 @@ func TestDSLGzipSerialize(t *testing.T) {
require.Equal(t, "hello world", data.(string), "could not get gzip encoded data")
}
func TestTimeToStringDSLFunction(t *testing.T) {
compiled, err := govaluate.NewEvaluableExpressionWithFunctions("timetostring(data)", HelperFunctions())
require.Nil(t, err, "could not compile encoder")
func TestDateTimeDSLFunction(t *testing.T) {
testDateTimeFormat := func(t *testing.T, dateTimeFormat string, dateTimeFunction *govaluate.EvaluableExpression, expectedFormattedTime string, currentUnixTime int64) {
dslFunctionParameters := map[string]interface{}{"dateTimeFormat": dateTimeFormat}
if currentUnixTime != 0 {
dslFunctionParameters["unixTime"] = currentUnixTime
}
result, err := dateTimeFunction.Evaluate(dslFunctionParameters)
data := time.Now()
result, err := compiled.Evaluate(map[string]interface{}{"data": data})
require.Nil(t, err, "could not evaluate compare time")
require.Equal(t, data.String(), result.(string), "could not get correct time format string")
require.Equal(t, expectedFormattedTime, result.(string), "could not get correct time format string")
}
t.Run("with Unix time", func(t *testing.T) {
dateTimeFunction, err := govaluate.NewEvaluableExpressionWithFunctions("date_time(dateTimeFormat)", HelperFunctions())
require.Nil(t, err, "could not compile encoder")
currentTime := time.Now()
expectedFormattedTime := currentTime.Format("02-01-2006 15:04")
testDateTimeFormat(t, "02-01-2006 15:04", dateTimeFunction, expectedFormattedTime, 0)
testDateTimeFormat(t, "%D-%M-%Y %H:%m", dateTimeFunction, expectedFormattedTime, 0)
})
t.Run("without Unix time", func(t *testing.T) {
dateTimeFunction, err := govaluate.NewEvaluableExpressionWithFunctions("date_time(dateTimeFormat, unixTime)", HelperFunctions())
require.Nil(t, err, "could not compile encoder")
currentTime := time.Now()
currentUnixTime := currentTime.Unix()
expectedFormattedTime := currentTime.Format("02-01-2006 15:04")
testDateTimeFormat(t, "02-01-2006 15:04", dateTimeFunction, expectedFormattedTime, currentUnixTime)
testDateTimeFormat(t, "%D-%M-%Y %H:%m", dateTimeFunction, expectedFormattedTime, currentUnixTime)
})
}
func TestDslFunctionSignatures(t *testing.T) {
type testCase struct {
methodName string
@ -111,15 +139,17 @@ func TestGetPrintableDslFunctionSignatures(t *testing.T) {
compare_versions(firstVersion, constraints ...string) bool
concat(args ...interface{}) string
contains(arg1, arg2 interface{}) interface{}
date(arg1 interface{}) interface{}
date_time(dateTimeFormat string, optionalUnixTime interface{}) string
dec_to_hex(arg1 interface{}) interface{}
generate_java_gadget(arg1, arg2, arg3 interface{}) interface{}
gzip(arg1 interface{}) interface{}
gzip_decode(arg1 interface{}) interface{}
hex_decode(arg1 interface{}) interface{}
hex_encode(arg1 interface{}) interface{}
hmac(arg1, arg2, arg3 interface{}) interface{}
html_escape(arg1 interface{}) interface{}
html_unescape(arg1 interface{}) interface{}
join(separator string, elements ...interface{}) string
len(arg1 interface{}) interface{}
md5(arg1 interface{}) interface{}
mmh3(arg1 interface{}) interface{}
@ -139,8 +169,6 @@ func TestGetPrintableDslFunctionSignatures(t *testing.T) {
reverse(arg1 interface{}) interface{}
sha1(arg1 interface{}) interface{}
sha256(arg1 interface{}) interface{}
time(arg1 interface{}) interface{}
timetostring(arg1 interface{}) interface{}
to_lower(arg1 interface{}) interface{}
to_number(arg1 interface{}) interface{}
to_string(arg1 interface{}) interface{}
@ -180,14 +208,17 @@ func TestDslExpressions(t *testing.T) {
`hex_encode("aa")`: "6161",
`html_escape("<body>test</body>")`: "&lt;body&gt;test&lt;/body&gt;",
`html_unescape("&lt;body&gt;test&lt;/body&gt;")`: "<body>test</body>",
`date("%Y-%M-%D")`: fmt.Sprintf("%02d-%02d-%02d", now.Year(), now.Month(), now.Day()),
`time("%H-%M")`: fmt.Sprintf("%02d-%02d", now.Hour(), now.Minute()),
`date_time("%Y-%M-%D")`: fmt.Sprintf("%02d-%02d-%02d", now.Year(), now.Month(), now.Day()),
`date_time("%Y-%M-%D", unix_time())`: fmt.Sprintf("%02d-%02d-%02d", now.Year(), now.Month(), now.Day()),
`date_time("%H-%m")`: fmt.Sprintf("%02d-%02d", now.Hour(), now.Minute()),
`date_time("02-01-2006 15:04", unix_time())`: now.Format("02-01-2006 15:04"),
`md5("Hello")`: "8b1a9953c4611296a827abf8c47804d7",
`md5(1234)`: "81dc9bdb52d04dc20036dbd8313ed055",
`mmh3("Hello")`: "316307400",
`remove_bad_chars("abcd", "bc")`: "ad",
`replace("Hello", "He", "Ha")`: "Hallo",
`concat("Hello", 123, "world")`: "Hello123world",
`join("_", "Hello", 123, "world")`: "Hello_123_world",
`repeat("a", 5)`: "aaaaa",
`repeat("a", "5")`: "aaaaa",
`repeat("../", "5")`: "../../../../../",
@ -226,6 +257,8 @@ func TestDslExpressions(t *testing.T) {
`compare_versions('v1.1.1', '>v1.1.0')`: true,
`compare_versions('v1.0.0', '>v0.0.1,<v1.0.1')`: true,
`compare_versions('v1.0.0', '>v0.0.1', '<v1.0.1')`: true,
`hmac('sha1', 'test', 'scrt')`: "8856b111056d946d5c6c92a21b43c233596623c6",
`hmac('sha256', 'test', 'scrt')`: "1f1bff5574f18426eb376d6dd5368a754e67a798aa2074644d5e3fd4c90c7a92",
}
for dslExpression, expectedResult := range dslExpressions {

View File

@ -12,7 +12,7 @@ func TestVariablesEvaluate(t *testing.T) {
data := `a1: "{{rand_base(5)}}"
a2: "{{md5(a1)}}"
a3: "this_is_random_text"
a4: "{{date('%Y-%M-%D')}}"
a4: "{{date_time('%Y-%M-%D')}}"
a5: "{{reverse(hostname)}}"
a6: "123456"`