diff --git a/v2/pkg/protocols/common/expressions/expressions.go b/v2/pkg/protocols/common/expressions/expressions.go new file mode 100644 index 00000000..427ba1a2 --- /dev/null +++ b/v2/pkg/protocols/common/expressions/expressions.go @@ -0,0 +1,39 @@ +package expressions + +import ( + "regexp" + + "github.com/Knetic/govaluate" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" +) + +var templateExpressionRegex = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`) + +// Evaluate checks if the match contains a dynamic variable, for each +// found one we will check if it's an expression and can +// be compiled, it will be evaluated and the results will be returned. +// +// The provided keys from finalValues will be used as variable names +// for substitution inside the expression. +func Evaluate(data string, base map[string]interface{}) (string, error) { + data = replacer.Replace(data, base) + + dynamicValues := make(map[string]interface{}) + for _, match := range templateExpressionRegex.FindAllString(data, -1) { + expr := generators.TrimDelimiters(match) + + compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) + if err != nil { + return "", err + } + result, err := compiled.Evaluate(base) + if err != nil { + return "", err + } + dynamicValues[expr] = result // convert x() => + } + // Replacer dynamic values if any in raw request and parse it + return replacer.Replace(data, dynamicValues), nil +} diff --git a/v2/pkg/protocols/common/expressions/expressions_test.go b/v2/pkg/protocols/common/expressions/expressions_test.go new file mode 100644 index 00000000..581138eb --- /dev/null +++ b/v2/pkg/protocols/common/expressions/expressions_test.go @@ -0,0 +1,25 @@ +package expressions + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEvaluate(t *testing.T) { + items := []struct { + input string + expected string + extra map[string]interface{} + }{ + {input: "{{hex_encode('PING')}}", expected: "50494e47", extra: map[string]interface{}{}}, + {input: "test", expected: "test", extra: map[string]interface{}{}}, + {input: "{{hex_encode(Item)}}", expected: "50494e47", extra: map[string]interface{}{"Item": "PING"}}, + } + for _, item := range items { + value, err := Evaluate(item.input, item.extra) + require.Nil(t, err, "could not evaluate helper") + + require.Equal(t, item.expected, value, "could not get correct expression") + } +} diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 7a4f6673..dfb4f717 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -11,8 +11,8 @@ import ( "strings" "time" - "github.com/Knetic/govaluate" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race" @@ -22,8 +22,7 @@ import ( ) var ( - urlWithPortRegex = regexp.MustCompile(`{{BaseURL}}:(\d+)`) - templateExpressionRegex = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`) + urlWithPortRegex = regexp.MustCompile(`{{BaseURL}}:(\d+)`) ) // generatedRequest is a single wrapped generated request for a template request @@ -122,31 +121,13 @@ func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest, // Combine the template payloads along with base // request values. finalValues := generators.MergeMaps(generatorValues, values) - rawRequest = replacer.Replace(rawRequest, finalValues) - // Check if the match contains a dynamic variable, for each - // found one we will check if it's an expression and can - // be compiled, it will be evaluated and the results will be returned. - // - // The provided keys from finalValues will be used as variable names - // for substitution inside the expression. - dynamicValues := make(map[string]interface{}) - for _, match := range templateExpressionRegex.FindAllString(rawRequest, -1) { - expr := generators.TrimDelimiters(match) - - compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) - if err != nil { - return nil, err - } - result, err := compiled.Evaluate(finalValues) - if err != nil { - return nil, err - } - dynamicValues[expr] = result // convert base64() => + // Evaulate the expressions for raw request if any. + var err error + rawRequest, err = expressions.Evaluate(rawRequest, finalValues) + if err != nil { + return nil, errors.Wrap(err, "could not evaluate helper expressions") } - - // Replacer dynamic values if any in raw request and parse it - rawRequest = replacer.Replace(rawRequest, dynamicValues) rawRequestData, err := raw.Parse(rawRequest, baseURL, r.request.Unsafe) if err != nil { return nil, err diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 8ace3e43..5d6f1ca2 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" ) @@ -75,6 +76,15 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { r.addresses = append(r.addresses, addressKV{ip: address, tls: shouldUseTLS}) } } + // Pre-compile any input dsl functions before executing the request. + for _, input := range r.Inputs { + if input.Type != "" { + continue + } + if compiled, err := expressions.Evaluate(input.Data, map[string]interface{}{}); err == nil { + input.Data = string(compiled) + } + } // Create a client for the class client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 04dfa10a..b3dee18f 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -3,6 +3,7 @@ package network import ( "context" "encoding/hex" + "io" "net" "net/url" "strings" @@ -74,16 +75,11 @@ func (r *Request) executeAddress(actualAddress, address, input string, shouldUse return errors.Wrap(err, "could not connect to server request") } defer conn.Close() - conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + conn.SetReadDeadline(time.Now().Add(time.Duration(r.options.Options.Timeout) * time.Second)) responseBuilder := &strings.Builder{} reqBuilder := &strings.Builder{} - // Read some data if any in the buffer - buffer := make([]byte, r.ReadSize) - n, _ := conn.Read(buffer) - responseBuilder.Write(buffer[:n]) - for _, input := range r.Inputs { var data []byte @@ -134,7 +130,12 @@ func (r *Request) executeAddress(actualAddress, address, input string, shouldUse bufferSize = r.ReadSize } final := make([]byte, bufferSize) - n, _ = conn.Read(final) + n, err := conn.Read(final) + if err != nil && err != io.EOF { + r.options.Output.Request(r.options.TemplateID, address, "network", err) + r.options.Progress.DecrementRequests(1) + return errors.Wrap(err, "could not read from server") + } responseBuilder.Write(final[:n]) if r.options.Options.Debug || r.options.Options.DebugResponse {