From 0e8270c7b568d557949ace46e22d196abd063c7f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 9 Jan 2022 12:52:04 +0100 Subject: [PATCH] Replacing expression regex with lexical analyzer (#1440) * Replacing regex with lexical analyzer taken from https://github.com/projectdiscovery/nuclei/blob/610beb8534e0701592bb5dd44355a0db3337cace/v2/pkg/protocols/common/expressions/expressions.go#L66 --- .../common/expressions/expressions.go | 18 ++++++++++++++---- .../common/expressions/expressions_test.go | 3 +++ v2/pkg/protocols/common/generators/maps.go | 4 +++- v2/pkg/protocols/common/marker/marker.go | 10 ++++++++++ v2/pkg/protocols/common/replacer/replacer.go | 16 +++++----------- .../protocols/headless/engine/page_actions.go | 3 ++- 6 files changed, 37 insertions(+), 17 deletions(-) create mode 100644 v2/pkg/protocols/common/marker/marker.go diff --git a/v2/pkg/protocols/common/expressions/expressions.go b/v2/pkg/protocols/common/expressions/expressions.go index 4f3a4b61..742acc91 100644 --- a/v2/pkg/protocols/common/expressions/expressions.go +++ b/v2/pkg/protocols/common/expressions/expressions.go @@ -1,17 +1,16 @@ package expressions import ( - "regexp" + "strings" "github.com/Knetic/govaluate" "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" ) -var templateExpressionRegex = regexp.MustCompile(`(?m){{[^}]+}}["')}]*`) - // Evaluate checks if the match contains a dynamic variable, for each // found one we will check if it's an expression and can // be compiled, it will be evaluated and the results will be returned. @@ -37,7 +36,7 @@ func evaluate(data string, base map[string]interface{}) (string, error) { data = replacer.Replace(data, base) dynamicValues := make(map[string]interface{}) - for _, match := range templateExpressionRegex.FindAllString(data, -1) { + for _, match := range findMatches(data) { expr := generators.TrimDelimiters(match) compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) @@ -53,3 +52,14 @@ func evaluate(data string, base map[string]interface{}) (string, error) { // Replacer dynamic values if any in raw request and parse it return replacer.Replace(data, dynamicValues), nil } + +func findMatches(data string) []string { + var matches []string + for _, token := range strings.Split(data, marker.ParenthesisOpen) { + closingToken := strings.LastIndex(token, marker.ParenthesisClose) + if closingToken > 0 { + matches = append(matches, token[:closingToken]) + } + } + return matches +} diff --git a/v2/pkg/protocols/common/expressions/expressions_test.go b/v2/pkg/protocols/common/expressions/expressions_test.go index 9fdc7c47..f2d5ec1c 100644 --- a/v2/pkg/protocols/common/expressions/expressions_test.go +++ b/v2/pkg/protocols/common/expressions/expressions_test.go @@ -12,12 +12,15 @@ func TestEvaluate(t *testing.T) { expected string extra map[string]interface{} }{ + {input: "{{url_encode('test}aaa')}}", expected: "test%7Daaa", extra: map[string]interface{}{}}, {input: "{{hex_encode('PING')}}", expected: "50494e47", extra: map[string]interface{}{}}, {input: "test", expected: "test", extra: map[string]interface{}{}}, {input: "{{hex_encode(Item)}}", expected: "50494e47", extra: map[string]interface{}{"Item": "PING"}}, {input: "{{hex_encode(Item)}}\r\n", expected: "50494e47\r\n", extra: map[string]interface{}{"Item": "PING"}}, {input: "{{someTestData}}{{hex_encode('PING')}}", expected: "{{someTestData}}50494e47", extra: map[string]interface{}{}}, {input: `_IWP_JSON_PREFIX_{{base64("{\"iwp_action\":\"add_site\",\"params\":{\"username\":\"\"}}")}}`, expected: "_IWP_JSON_PREFIX_eyJpd3BfYWN0aW9uIjoiYWRkX3NpdGUiLCJwYXJhbXMiOnsidXNlcm5hbWUiOiIifX0=", extra: map[string]interface{}{}}, + {input: "{{}}", expected: "{{}}", extra: map[string]interface{}{}}, + {input: `"{{hex_encode('PING')}}"`, expected: `"50494e47"`, extra: map[string]interface{}{}}, } for _, item := range items { value, err := Evaluate(item.input, item.extra) diff --git a/v2/pkg/protocols/common/generators/maps.go b/v2/pkg/protocols/common/generators/maps.go index d88ec3db..bdd1ff66 100644 --- a/v2/pkg/protocols/common/generators/maps.go +++ b/v2/pkg/protocols/common/generators/maps.go @@ -3,6 +3,8 @@ package generators import ( "reflect" "strings" + + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker" ) // MergeMapsMany merges many maps into a new map @@ -85,5 +87,5 @@ func CopyMapWithDefaultValue(originalMap map[string][]string, defaultValue inter // TrimDelimiters removes trailing brackets func TrimDelimiters(s string) string { - return strings.TrimSuffix(strings.TrimPrefix(s, "{{"), "}}") + return strings.TrimSuffix(strings.TrimPrefix(s, marker.ParenthesisOpen), marker.ParenthesisClose) } diff --git a/v2/pkg/protocols/common/marker/marker.go b/v2/pkg/protocols/common/marker/marker.go new file mode 100644 index 00000000..d0790a12 --- /dev/null +++ b/v2/pkg/protocols/common/marker/marker.go @@ -0,0 +1,10 @@ +package marker + +const ( + // General marker (open/close) + General = "§" + // ParenthesisOpen marker - begin of a placeholder + ParenthesisOpen = "{{" + // ParenthesisClose marker - end of a placeholder + ParenthesisClose = "}}" +) diff --git a/v2/pkg/protocols/common/replacer/replacer.go b/v2/pkg/protocols/common/replacer/replacer.go index e91ed5e9..9cb44823 100644 --- a/v2/pkg/protocols/common/replacer/replacer.go +++ b/v2/pkg/protocols/common/replacer/replacer.go @@ -3,32 +3,26 @@ package replacer import ( "strings" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) -// Payload marker constants -const ( - MarkerGeneral = "§" - MarkerParenthesisOpen = "{{" - MarkerParenthesisClose = "}}" -) - // Replace replaces placeholders in template with values on the fly. func Replace(template string, values map[string]interface{}) string { var replacerItems []string builder := &strings.Builder{} for key, val := range values { - builder.WriteString(MarkerParenthesisOpen) + builder.WriteString(marker.ParenthesisOpen) builder.WriteString(key) - builder.WriteString(MarkerParenthesisClose) + builder.WriteString(marker.ParenthesisClose) replacerItems = append(replacerItems, builder.String()) builder.Reset() replacerItems = append(replacerItems, types.ToString(val)) - builder.WriteString(MarkerGeneral) + builder.WriteString(marker.General) builder.WriteString(key) - builder.WriteString(MarkerGeneral) + builder.WriteString(marker.General) replacerItems = append(replacerItems, builder.String()) builder.Reset() replacerItems = append(replacerItems, types.ToString(val)) diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index b186dcc5..54505c77 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -14,6 +14,7 @@ import ( "github.com/go-rod/rod/lib/proto" "github.com/go-rod/rod/lib/utils" "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker" "github.com/segmentio/ksuid" "github.com/valyala/fasttemplate" ) @@ -254,7 +255,7 @@ func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.UR parsedString := parsed.String() values["BaseURL"] = parsedString - final := fasttemplate.ExecuteStringStd(URL, "{{", "}}", values) + final := fasttemplate.ExecuteStringStd(URL, marker.ParenthesisOpen, marker.ParenthesisClose, values) if err := p.page.Navigate(final); err != nil { return errors.Wrap(err, "could not navigate") }