daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implementing lexer with runtime expression validation

dev
mzack 2022-01-19 14:10:11 +01:00
parent 6a5b9a4337
commit 369255a4fe
4 changed files with 120 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
v2/go.mod
View File

@ -38,7 +38,7 @@ require (
github.com/projectdiscovery/rawhttp v0.0.7
github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660
github.com/projectdiscovery/retryablehttp-go v1.0.2
github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9
github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700
github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.3.0 // indirect

3
v2/go.sum
View File

@ -457,8 +457,9 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjB
github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes=
github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700 h1:L7Vb5AdzIV1Xs088Nvslfhh/piKP9gjTxjxfiqnd4mk=
github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24=
github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8=
github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6/go.mod h1:8OfZj8p/axkUM/TJoS/O9LDjj/S8u17rxRbqluE9CU4=

121
v2/pkg/protocols/common/expressions/expressions.go
View File

@ -6,9 +6,9 @@ import (
"github.com/Knetic/govaluate"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
"github.com/projectdiscovery/stringsutil"
)
// Evaluate checks if the match contains a dynamic variable, for each
@ -33,13 +33,17 @@ func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) {
}
func evaluate(data string, base map[string]interface{}) (string, error) {
data = replacer.Replace(data, base)
// expressions can be:
// - simple: containing base values keys (variables)
// - complex: containing helper functions [ + variables]
// literals like {{2+2}} are not considered expressions
expressions := findExpressions(data, mergeFunctions(dsl.HelperFunctions(), mapToFunctions(base)))
dynamicValues := make(map[string]interface{})
for _, match := range findMatches(data) {
expr := generators.TrimDelimiters(match)
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
for _, expression := range expressions {
// replace variable placeholders with base values
expression = replacer.Replace(expression, base)
// turns expressions (either helper functions+base values or base values)
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions())
if err != nil {
continue
}
@ -47,19 +51,104 @@ func evaluate(data string, base map[string]interface{}) (string, error) {
if err != nil {
continue
}
dynamicValues[expr] = result
dynamicValues[expression] = result
}
// Replacer dynamic values if any in raw request and parse it
// Replacer dynamic values if any in raw request and parse it
return replacer.Replace(data, dynamicValues), nil
}
func findMatches(data string) []string {
var matches []string
for _, token := range strings.Split(data, marker.ParenthesisOpen) {
closingToken := strings.LastIndex(token, marker.ParenthesisClose)
if closingToken > 0 {
matches = append(matches, token[:closingToken])
// maxIterations to avoid infinite loop
const maxIterations = 250
func findExpressions(data string, functions map[string]govaluate.ExpressionFunction) []string {
var (
iterations int
exps []string
)
for {
// check if we reached the maximum number of iterations
if iterations > maxIterations {
break
}
iterations++
// attempt to find open markers
indexOpenMarker := strings.Index(data, marker.ParenthesisOpen)
// exits if not found
if indexOpenMarker < 0 {
break
}
indexOpenMarkerOffset := indexOpenMarker + len(marker.ParenthesisOpen)
shouldSearchCloseMarker := true
closeMarkerFound := false
innerData := data
var potentialMatch string
var indexCloseMarker, indexCloseMarkerOffset int
skip := indexOpenMarkerOffset
for shouldSearchCloseMarker {
// attempt to find close marker
indexCloseMarker = stringsutil.IndexAt(innerData, marker.ParenthesisClose, skip)
// if no close markers are found exit
if indexCloseMarker < 0 {
shouldSearchCloseMarker = false
continue
}
indexCloseMarkerOffset = indexCloseMarker + len(marker.ParenthesisClose)
potentialMatch = innerData[indexOpenMarkerOffset:indexCloseMarker]
if isExpression(potentialMatch, functions) {
closeMarkerFound = true
shouldSearchCloseMarker = false
exps = append(exps, potentialMatch)
} else {
skip = indexCloseMarkerOffset
}
}
if closeMarkerFound {
// move after the close marker
data = data[indexCloseMarkerOffset:]
} else {
// move after the open marker
data = data[indexOpenMarkerOffset:]
}
}
return matches
return exps
}
func isExpression(data string, functions map[string]govaluate.ExpressionFunction) bool {
if _, err := govaluate.NewEvaluableExpression(data); err == nil {
return stringsutil.ContainsAny(data, getFunctionsNames(functions)...)
}
// check if it's a complex expression
_, err := govaluate.NewEvaluableExpressionWithFunctions(data, dsl.HelperFunctions())
return err == nil
}
func mapToFunctions(vars map[string]interface{}) map[string]govaluate.ExpressionFunction {
f := make(map[string]govaluate.ExpressionFunction)
for k := range vars {
f[k] = nil
}
return f
}
func mergeFunctions(m ...map[string]govaluate.ExpressionFunction) map[string]govaluate.ExpressionFunction {
o := make(map[string]govaluate.ExpressionFunction)
for _, mm := range m {
for k, v := range mm {
o[k] = v
}
}
return o
}
func getFunctionsNames(m map[string]govaluate.ExpressionFunction) []string {
var keys []string
for k := range m {
keys = append(keys, k)
}
return keys
}

24
v2/pkg/protocols/common/expressions/expressions_test.go
View File

@ -14,19 +14,19 @@ func TestEvaluate(t *testing.T) {
}{
{input: "{{url_encode('test}aaa')}}", expected: "test%7Daaa", extra: map[string]interface{}{}},
{input: "{{hex_encode('PING')}}", expected: "50494e47", extra: map[string]interface{}{}},
// TODO #1501
//{input: "{{hex_encode('{{')}}", expected: "7b7b", extra: map[string]interface{}{}},
//{input: `{{concat("{{", 123, "*", 123, "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
//{input: `{{concat("{{", "123*123", "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
//{input: `{{"{{" + '123*123' + "}}"}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
{input: "{{hex_encode('{{')}}", expected: "7b7b", extra: map[string]interface{}{}},
{input: `{{concat("{{", 123, "*", 123, "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
{input: `{{concat("{{", "123*123", "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
{input: `{{"{{" + '123*123' + "}}"}}`, expected: `{{"{{" + '123*123' + "}}"}}`, extra: map[string]interface{}{}},
{input: `{{a + '123*123' + b}}`, expected: `aa123*123bb`, extra: map[string]interface{}{"a": "aa", "b": "bb"}},
{input: `{{concat(123,'*',123)}}`, expected: "123*123", extra: map[string]interface{}{}},
{input: `{{1+1}}`, expected: "2", extra: map[string]interface{}{}},
{input: `{{"1"+"1"}}`, expected: "11", extra: map[string]interface{}{}},
{input: `{{"1" + '*' + "1"}}`, expected: "1*1", extra: map[string]interface{}{}},
{input: `{{"a" + 'b' + "c"}}`, expected: "abc", extra: map[string]interface{}{}},
{input: `{{10*2}}`, expected: "20", extra: map[string]interface{}{}},
{input: `{{10/2}}`, expected: "5", extra: map[string]interface{}{}},
{input: `{{10-2}}`, expected: "8", extra: map[string]interface{}{}},
{input: `{{1+1}}`, expected: "{{1+1}}", extra: map[string]interface{}{}},
{input: `{{"1"+"1"}}`, expected: `{{"1"+"1"}}`, extra: map[string]interface{}{}},
{input: `{{"1" + '*' + "1"}}`, expected: `{{"1" + '*' + "1"}}`, extra: map[string]interface{}{}},
{input: `{{"a" + 'b' + "c"}}`, expected: `{{"a" + 'b' + "c"}}`, extra: map[string]interface{}{}},
{input: `{{10*2}}`, expected: `{{10*2}}`, extra: map[string]interface{}{}},
{input: `{{10/2}}`, expected: `{{10/2}}`, extra: map[string]interface{}{}},
{input: `{{10-2}}`, expected: `{{10-2}}`, extra: map[string]interface{}{}},
{input: "test", expected: "test", extra: map[string]interface{}{}},
{input: "{{hex_encode(Item)}}", expected: "50494e47", extra: map[string]interface{}{"Item": "PING"}},
{input: "{{hex_encode(Item)}}\r\n", expected: "50494e47\r\n", extra: map[string]interface{}{"Item": "PING"}},