Added expressions package + expressions support to network protocols

v2/pkg/protocols/common/expressions/expressions.go

@@ -0,0 +1,39 @@
+package expressions
+
+import (
+	"regexp"
+
+	"github.com/Knetic/govaluate"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
+)
+
+var templateExpressionRegex = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`)
+
+// Evaluate checks if the match contains a dynamic variable, for each
+// found one we will check if it's an expression and can
+// be compiled, it will be evaluated and the results will be returned.
+//
+// The provided keys from finalValues will be used as variable names
+// for substitution inside the expression.
+func Evaluate(data string, base map[string]interface{}) (string, error) {
+	data = replacer.Replace(data, base)
+
+	dynamicValues := make(map[string]interface{})
+	for _, match := range templateExpressionRegex.FindAllString(data, -1) {
+		expr := generators.TrimDelimiters(match)
+
+		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
+		if err != nil {
+			return "", err
+		}
+		result, err := compiled.Evaluate(base)
+		if err != nil {
+			return "", err
+		}
+		dynamicValues[expr] = result // convert x(<payload_name>) => <x-representation>
+	}
+	// Replacer dynamic values if any in raw request and parse it
+	return replacer.Replace(data, dynamicValues), nil
+}

v2/pkg/protocols/common/expressions/expressions_test.go

@@ -0,0 +1,25 @@
+package expressions
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestEvaluate(t *testing.T) {
+	items := []struct {
+		input    string
+		expected string
+		extra    map[string]interface{}
+	}{
+		{input: "{{hex_encode('PING')}}", expected: "50494e47", extra: map[string]interface{}{}},
+		{input: "test", expected: "test", extra: map[string]interface{}{}},
+		{input: "{{hex_encode(Item)}}", expected: "50494e47", extra: map[string]interface{}{"Item": "PING"}},
+	}
+	for _, item := range items {
+		value, err := Evaluate(item.input, item.extra)
+		require.Nil(t, err, "could not evaluate helper")
+
+		require.Equal(t, item.expected, value, "could not get correct expression")
+	}
+}

v2/pkg/protocols/http/build_request.go

@@ -11,8 +11,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/Knetic/govaluate"
-	"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race"
@@ -22,8 +22,7 @@ import (
 )
 
 var (
-	urlWithPortRegex        = regexp.MustCompile(`{{BaseURL}}:(\d+)`)
-	templateExpressionRegex = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`)
+	urlWithPortRegex = regexp.MustCompile(`{{BaseURL}}:(\d+)`)
 )
 
 // generatedRequest is a single wrapped generated request for a template request
@@ -122,31 +121,13 @@ func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest,
 	// Combine the template payloads along with base
 	// request values.
 	finalValues := generators.MergeMaps(generatorValues, values)
-	rawRequest = replacer.Replace(rawRequest, finalValues)
 
-	// Check if the match contains a dynamic variable, for each
-	// found one we will check if it's an expression and can
-	// be compiled, it will be evaluated and the results will be returned.
-	//
-	// The provided keys from finalValues will be used as variable names
-	// for substitution inside the expression.
-	dynamicValues := make(map[string]interface{})
-	for _, match := range templateExpressionRegex.FindAllString(rawRequest, -1) {
-		expr := generators.TrimDelimiters(match)
-
-		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
-		if err != nil {
-			return nil, err
-		}
-		result, err := compiled.Evaluate(finalValues)
-		if err != nil {
-			return nil, err
-		}
-		dynamicValues[expr] = result // convert base64(<payload_name>) => <base64-representation>
+	// Evaulate the expressions for raw request if any.
+	var err error
+	rawRequest, err = expressions.Evaluate(rawRequest, finalValues)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not evaluate helper expressions")
 	}
-
-	// Replacer dynamic values if any in raw request and parse it
-	rawRequest = replacer.Replace(rawRequest, dynamicValues)
 	rawRequestData, err := raw.Parse(rawRequest, baseURL, r.request.Unsafe)
 	if err != nil {
 		return nil, err

v2/pkg/protocols/network/network.go

@@ -8,6 +8,7 @@ import (
 	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
 )
 
@@ -75,6 +76,15 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
 			r.addresses = append(r.addresses, addressKV{ip: address, tls: shouldUseTLS})
 		}
 	}
+	// Pre-compile any input dsl functions before executing the request.
+	for _, input := range r.Inputs {
+		if input.Type != "" {
+			continue
+		}
+		if compiled, err := expressions.Evaluate(input.Data, map[string]interface{}{}); err == nil {
+			input.Data = string(compiled)
+		}
+	}
 
 	// Create a client for the class
 	client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{})

v2/pkg/protocols/network/request.go

@@ -3,6 +3,7 @@ package network
 import (
 	"context"
 	"encoding/hex"
+	"io"
 	"net"
 	"net/url"
 	"strings"
@@ -74,16 +75,11 @@ func (r *Request) executeAddress(actualAddress, address, input string, shouldUse
 		return errors.Wrap(err, "could not connect to server request")
 	}
 	defer conn.Close()
-	conn.SetReadDeadline(time.Now().Add(5 * time.Second))
+	conn.SetReadDeadline(time.Now().Add(time.Duration(r.options.Options.Timeout) * time.Second))
 
 	responseBuilder := &strings.Builder{}
 	reqBuilder := &strings.Builder{}
 
-	// Read some data if any in the buffer
-	buffer := make([]byte, r.ReadSize)
-	n, _ := conn.Read(buffer)
-	responseBuilder.Write(buffer[:n])
-
 	for _, input := range r.Inputs {
 		var data []byte
 
@@ -134,7 +130,12 @@ func (r *Request) executeAddress(actualAddress, address, input string, shouldUse
 		bufferSize = r.ReadSize
 	}
 	final := make([]byte, bufferSize)
-	n, _ = conn.Read(final)
+	n, err := conn.Read(final)
+	if err != nil && err != io.EOF {
+		r.options.Output.Request(r.options.TemplateID, address, "network", err)
+		r.options.Progress.DecrementRequests(1)
+		return errors.Wrap(err, "could not read from server")
+	}
 	responseBuilder.Write(final[:n])
 
 	if r.options.Options.Debug || r.options.Options.DebugResponse {