Replacing expression regex with lexical analyzer (#1440)

* Replacing regex with lexical analyzer taken from 610beb8534/v2/pkg/protocols/common/expressions/expressions.go (L66)

v2/pkg/protocols/common/expressions/expressions.go

@@ -1,17 +1,16 @@
 package expressions
 
 import (
-	"regexp"
+	"strings"
 
 	"github.com/Knetic/govaluate"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
 )
 
-var templateExpressionRegex = regexp.MustCompile(`(?m){{[^}]+}}["')}]*`)
-
 // Evaluate checks if the match contains a dynamic variable, for each
 // found one we will check if it's an expression and can
 // be compiled, it will be evaluated and the results will be returned.
@@ -37,7 +36,7 @@ func evaluate(data string, base map[string]interface{}) (string, error) {
 	data = replacer.Replace(data, base)
 
 	dynamicValues := make(map[string]interface{})
-	for _, match := range templateExpressionRegex.FindAllString(data, -1) {
+	for _, match := range findMatches(data) {
 		expr := generators.TrimDelimiters(match)
 
 		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
@@ -53,3 +52,14 @@ func evaluate(data string, base map[string]interface{}) (string, error) {
 	// Replacer dynamic values if any in raw request and parse  it
 	return replacer.Replace(data, dynamicValues), nil
 }
+
+func findMatches(data string) []string {
+	var matches []string
+	for _, token := range strings.Split(data, marker.ParenthesisOpen) {
+		closingToken := strings.LastIndex(token, marker.ParenthesisClose)
+		if closingToken > 0 {
+			matches = append(matches, token[:closingToken])
+		}
+	}
+	return matches
+}

v2/pkg/protocols/common/expressions/expressions_test.go

@@ -12,12 +12,15 @@ func TestEvaluate(t *testing.T) {
 		expected string
 		extra    map[string]interface{}
 	}{
+		{input: "{{url_encode('test}aaa')}}", expected: "test%7Daaa", extra: map[string]interface{}{}},
 		{input: "{{hex_encode('PING')}}", expected: "50494e47", extra: map[string]interface{}{}},
 		{input: "test", expected: "test", extra: map[string]interface{}{}},
 		{input: "{{hex_encode(Item)}}", expected: "50494e47", extra: map[string]interface{}{"Item": "PING"}},
 		{input: "{{hex_encode(Item)}}\r\n", expected: "50494e47\r\n", extra: map[string]interface{}{"Item": "PING"}},
 		{input: "{{someTestData}}{{hex_encode('PING')}}", expected: "{{someTestData}}50494e47", extra: map[string]interface{}{}},
 		{input: `_IWP_JSON_PREFIX_{{base64("{\"iwp_action\":\"add_site\",\"params\":{\"username\":\"\"}}")}}`, expected: "_IWP_JSON_PREFIX_eyJpd3BfYWN0aW9uIjoiYWRkX3NpdGUiLCJwYXJhbXMiOnsidXNlcm5hbWUiOiIifX0=", extra: map[string]interface{}{}},
+		{input: "{{}}", expected: "{{}}", extra: map[string]interface{}{}},
+		{input: `"{{hex_encode('PING')}}"`, expected: `"50494e47"`, extra: map[string]interface{}{}},
 	}
 	for _, item := range items {
 		value, err := Evaluate(item.input, item.extra)

v2/pkg/protocols/common/generators/maps.go

@@ -3,6 +3,8 @@ package generators
 import (
 	"reflect"
 	"strings"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker"
 )
 
 // MergeMapsMany merges many maps into a new map
@@ -85,5 +87,5 @@ func CopyMapWithDefaultValue(originalMap map[string][]string, defaultValue inter
 
 // TrimDelimiters removes trailing brackets
 func TrimDelimiters(s string) string {
-	return strings.TrimSuffix(strings.TrimPrefix(s, "{{"), "}}")
+	return strings.TrimSuffix(strings.TrimPrefix(s, marker.ParenthesisOpen), marker.ParenthesisClose)
 }

v2/pkg/protocols/common/marker/marker.go

@@ -0,0 +1,10 @@
+package marker
+
+const (
+	// General marker (open/close)
+	General = "§"
+	// ParenthesisOpen marker - begin of a placeholder
+	ParenthesisOpen = "{{"
+	// ParenthesisClose marker - end of a placeholder
+	ParenthesisClose = "}}"
+)

v2/pkg/protocols/common/replacer/replacer.go

@@ -3,32 +3,26 @@ package replacer
 import (
 	"strings"
 
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
-// Payload marker constants
-const (
-	MarkerGeneral          = "§"
-	MarkerParenthesisOpen  = "{{"
-	MarkerParenthesisClose = "}}"
-)
-
 // Replace replaces placeholders in template with values on the fly.
 func Replace(template string, values map[string]interface{}) string {
 	var replacerItems []string
 
 	builder := &strings.Builder{}
 	for key, val := range values {
-		builder.WriteString(MarkerParenthesisOpen)
+		builder.WriteString(marker.ParenthesisOpen)
 		builder.WriteString(key)
-		builder.WriteString(MarkerParenthesisClose)
+		builder.WriteString(marker.ParenthesisClose)
 		replacerItems = append(replacerItems, builder.String())
 		builder.Reset()
 		replacerItems = append(replacerItems, types.ToString(val))
 
-		builder.WriteString(MarkerGeneral)
+		builder.WriteString(marker.General)
 		builder.WriteString(key)
-		builder.WriteString(MarkerGeneral)
+		builder.WriteString(marker.General)
 		replacerItems = append(replacerItems, builder.String())
 		builder.Reset()
 		replacerItems = append(replacerItems, types.ToString(val))

v2/pkg/protocols/headless/engine/page_actions.go

@@ -14,6 +14,7 @@ import (
 	"github.com/go-rod/rod/lib/proto"
 	"github.com/go-rod/rod/lib/utils"
 	"github.com/pkg/errors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker"
 	"github.com/segmentio/ksuid"
 	"github.com/valyala/fasttemplate"
 )
@@ -254,7 +255,7 @@ func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.UR
 	parsedString := parsed.String()
 	values["BaseURL"] = parsedString
 
-	final := fasttemplate.ExecuteStringStd(URL, "{{", "}}", values)
+	final := fasttemplate.ExecuteStringStd(URL, marker.ParenthesisOpen, marker.ParenthesisClose, values)
 	if err := p.page.Navigate(final); err != nil {
 		return errors.Wrap(err, "could not navigate")
 	}