memory leak fixes and optimizations (#4680)

* feat http response memory optimization + reuse buffers

* update nuclei version

* feat: reuse js vm's and compile to programs

* fix failing http integration test

* remove dead code + add -jsc

* feat reuse js vms in pool with concurrency

* update comments as per review

* bug fix+ update interactsh test to look for dns interaction

* try enabling all interactsh integration tests

---------

Co-authored-by: mzack <marco.rivoli.nvh@gmail.com>

cmd/integration-test/interactsh.go

@@ -1,11 +1,9 @@
 package main
 
-import osutils "github.com/projectdiscovery/utils/os"
-
 // All Interactsh related testcases
 var interactshTestCases = []TestCaseInfo{
-	{Path: "protocols/http/interactsh.yaml", TestCase: &httpInteractshRequest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
-	{Path: "protocols/http/interactsh-stop-at-first-match.yaml", TestCase: &httpInteractshStopAtFirstMatchRequest{}, DisableOn: func() bool { return true }},
-	{Path: "protocols/http/default-matcher-condition.yaml", TestCase: &httpDefaultMatcherCondition{}, DisableOn: func() bool { return true }}, // disable this test for now
+	{Path: "protocols/http/interactsh.yaml", TestCase: &httpInteractshRequest{}, DisableOn: func() bool { return false }},
+	{Path: "protocols/http/interactsh-stop-at-first-match.yaml", TestCase: &httpInteractshStopAtFirstMatchRequest{}, DisableOn: func() bool { return false }}, // disable this test for now
+	{Path: "protocols/http/default-matcher-condition.yaml", TestCase: &httpDefaultMatcherCondition{}, DisableOn: func() bool { return false }},
 	{Path: "protocols/http/interactsh-requests-mc-and.yaml", TestCase: &httpInteractshRequestsWithMCAnd{}},
 }

cmd/nuclei/main.go

@@ -319,6 +319,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"),
 		flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"),
 		flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "headc", 10, "maximum number of headless templates to be executed in parallel"),
+		flagSet.IntVarP(&options.JsConcurrency, "js-concurrency", "jsc", 120, "maximum number of javascript runtimes to be executed in parallel"),
 	)
 	flagSet.CreateGroup("optimization", "Optimizations",
 		flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait in seconds before timeout"),

integration_tests/protocols/http/default-matcher-condition.yaml

@@ -15,7 +15,7 @@ requests:
       - type: word
         part: interactsh_protocol
         words:
-          - "http"
+          - "dns"
 
       - type: status
         status:

integration_tests/protocols/http/interactsh-stop-at-first-match.yaml

@@ -24,6 +24,6 @@ requests:
 
     matchers:
       - type: word
-        part: interactsh_protocol # Confirms the HTTP Interaction
+        part: interactsh_protocol # Confirms DNS Interaction
         words:
-          - "http"
+          - "dns"

integration_tests/protocols/http/interactsh.yaml

@@ -16,4 +16,4 @@ requests:
       - type: word
         part: interactsh_protocol # Confirms the HTTP Interaction
         words:
-          - "http"
+          - "dns"

pkg/catalog/config/constants.go

@@ -17,7 +17,7 @@ const (
 	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
-	Version = `v3.1.7`
+	Version = `v3.1.8-dev`
 	// Directory Names of custom templates
 	CustomS3TemplatesDirName     = "s3"
 	CustomGitHubTemplatesDirName = "github"

pkg/js/compiler/compiler.go

@@ -3,78 +3,34 @@ package compiler
 
 import (
 	"context"
-	"runtime/debug"
+	"fmt"
 	"time"
 
 	"github.com/dop251/goja"
-	"github.com/dop251/goja/parser"
-	"github.com/dop251/goja_nodejs/console"
-	"github.com/dop251/goja_nodejs/require"
-	jsoniter "github.com/json-iterator/go"
-	"github.com/pkg/errors"
 
-	"github.com/projectdiscovery/gologger"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libbytes"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libfs"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libikev2"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libkerberos"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libldap"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libmssql"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libmysql"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libnet"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/liboracle"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libpop3"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libpostgres"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/librdp"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libredis"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/librsync"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libsmb"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libsmtp"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libssh"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libstructs"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libtelnet"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libvnc"
-	"github.com/projectdiscovery/nuclei/v3/pkg/js/global"
-	"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/goconsole"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	contextutil "github.com/projectdiscovery/utils/context"
 )
 
 // Compiler provides a runtime to execute goja runtime
 // based javascript scripts efficiently while also
 // providing them access to custom modules defined in libs/.
-type Compiler struct {
-	registry *require.Registry
-}
+type Compiler struct{}
 
 // New creates a new compiler for the goja runtime.
 func New() *Compiler {
-	registry := new(require.Registry) // this can be shared by multiple runtimes
-	// autoregister console node module with default printer it uses gologger backend
-	require.RegisterNativeModule(console.ModuleName, console.RequireWithPrinter(goconsole.NewGoConsolePrinter()))
-	return &Compiler{registry: registry}
+	return &Compiler{}
 }
 
 // ExecuteOptions provides options for executing a script.
 type ExecuteOptions struct {
-	// Pool specifies whether to use a pool of goja runtimes
-	// Can be used to speedup execution but requires
-	// the script to not make any global changes.
-	Pool bool
-
-	// CaptureOutput specifies whether to capture the output
-	// of the script execution.
-	CaptureOutput bool
-
-	// CaptureVariables specifies the variables to capture
-	// from the script execution.
-	CaptureVariables []string
-
 	// Callback can be used to register new runtime helper functions
 	// ex: export etc
 	Callback func(runtime *goja.Runtime) error
 
+	// Cleanup is extra cleanup function to be called after execution
+	Cleanup func(runtime *goja.Runtime)
+
 	/// Timeout for this script execution
 	Timeout int
 }
@@ -111,51 +67,30 @@ func (e ExecuteResult) GetSuccess() bool {
 
 // Execute executes a script with the default options.
 func (c *Compiler) Execute(code string, args *ExecuteArgs) (ExecuteResult, error) {
-	return c.ExecuteWithOptions(code, args, &ExecuteOptions{})
-}
-
-// VM returns a new goja runtime for the compiler.
-func (c *Compiler) VM() *goja.Runtime {
-	runtime := c.newRuntime(false)
-	runtime.SetParserOptions(parser.WithDisableSourceMaps)
-	c.registerHelpersForVM(runtime)
-	return runtime
+	p, err := goja.Compile("", code, false)
+	if err != nil {
+		return nil, err
+	}
+	return c.ExecuteWithOptions(p, args, &ExecuteOptions{})
 }
 
 // ExecuteWithOptions executes a script with the provided options.
-func (c *Compiler) ExecuteWithOptions(code string, args *ExecuteArgs, opts *ExecuteOptions) (ExecuteResult, error) {
-	defer func() {
-		if err := recover(); err != nil {
-			gologger.Error().Msgf("Recovered panic %s %v: %v", code, args, err)
-			gologger.Verbose().Msgf("%s", debug.Stack())
-			return
-		}
-	}()
+func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (ExecuteResult, error) {
 	if opts == nil {
 		opts = &ExecuteOptions{}
 	}
-	runtime := c.newRuntime(opts.Pool)
-	c.registerHelpersForVM(runtime)
-
-	// register runtime functions if any
-	if opts.Callback != nil {
-		if err := opts.Callback(runtime); err != nil {
-			return nil, err
-		}
-	}
-
 	if args == nil {
 		args = NewExecuteArgs()
 	}
-	for k, v := range args.Args {
-		_ = runtime.Set(k, v)
-	}
+	// handle nil maps
 	if args.TemplateCtx == nil {
 		args.TemplateCtx = make(map[string]interface{})
 	}
+	if args.Args == nil {
+		args.Args = make(map[string]interface{})
+	}
 	// merge all args into templatectx
 	args.TemplateCtx = generators.MergeMaps(args.TemplateCtx, args.Args)
-	_ = runtime.Set("template", args.TemplateCtx)
 
 	if opts.Timeout <= 0 || opts.Timeout > 180 {
 		// some js scripts can take longer time so allow configuring timeout
@@ -170,72 +105,13 @@ func (c *Compiler) ExecuteWithOptions(code string, args *ExecuteArgs, opts *Exec
 	results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) {
 		defer func() {
 			if r := recover(); r != nil {
-				err = errors.Errorf("panic: %v", r)
+				err = fmt.Errorf("panic: %v", r)
 			}
 		}()
-		return runtime.RunString(code)
+		return executeProgram(program, args, opts)
 	})
 	if err != nil {
 		return nil, err
 	}
-	captured := results.Export()
-
-	if opts.CaptureOutput {
-		return convertOutputToResult(captured)
-	}
-	if len(opts.CaptureVariables) > 0 {
-		return c.captureVariables(runtime, opts.CaptureVariables)
-	}
-	// success is true by default . since js throws errors on failure
-	// hence output result is always success
-	return ExecuteResult{"response": captured, "success": results.ToBoolean()}, nil
-}
-
-// captureVariables captures the variables from the runtime.
-func (c *Compiler) captureVariables(runtime *goja.Runtime, variables []string) (ExecuteResult, error) {
-	results := make(ExecuteResult, len(variables))
-	for _, variable := range variables {
-		value := runtime.Get(variable)
-		if value == nil {
-			continue
-		}
-		results[variable] = value.Export()
-	}
-	return results, nil
-}
-
-func convertOutputToResult(output interface{}) (ExecuteResult, error) {
-	marshalled, err := jsoniter.Marshal(output)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not marshal output")
-	}
-
-	var outputMap map[string]interface{}
-	if err := jsoniter.Unmarshal(marshalled, &outputMap); err != nil {
-		var v interface{}
-		if unmarshalErr := jsoniter.Unmarshal(marshalled, &v); unmarshalErr != nil {
-			return nil, unmarshalErr
-		}
-		outputMap = map[string]interface{}{"output": v}
-		return outputMap, nil
-	}
-	return outputMap, nil
-}
-
-// newRuntime creates a new goja runtime
-// TODO: Add support for runtime reuse for helper functions
-func (c *Compiler) newRuntime(reuse bool) *goja.Runtime {
-	return protocolstate.NewJSRuntime()
-}
-
-// registerHelpersForVM registers all the helper functions for the goja runtime.
-func (c *Compiler) registerHelpersForVM(runtime *goja.Runtime) {
-	_ = c.registry.Enable(runtime)
-	// by default import below modules every time
-	_ = runtime.Set("console", require.Require(runtime, console.ModuleName))
-
-	// Register embedded scripts
-	if err := global.RegisterNativeScripts(runtime); err != nil {
-		gologger.Error().Msgf("Could not register scripts: %s\n", err)
-	}
+	return ExecuteResult{"response": results.Export(), "success": results.ToBoolean()}, nil
 }

pkg/js/compiler/compiler_test.go

@@ -38,36 +38,6 @@ func TestExecuteResultGetSuccess(t *testing.T) {
 	}
 }
 
-func TestCompilerCaptureVariables(t *testing.T) {
-	compiler := New()
-	result, err := compiler.ExecuteWithOptions("var a = 1;", NewExecuteArgs(), &ExecuteOptions{CaptureVariables: []string{"a"}})
-	if err != nil {
-		t.Fatal(err)
-	}
-	gotValue, ok := result["a"]
-	if !ok {
-		t.Fatalf("expected a to be present in the result")
-	}
-	if gotValue.(int64) != 1 {
-		t.Fatalf("expected a to be 1, got=%v", gotValue)
-	}
-}
-
-func TestCompilerCaptureOutput(t *testing.T) {
-	compiler := New()
-	result, err := compiler.ExecuteWithOptions("let obj = {'a':'b'}; obj", NewExecuteArgs(), &ExecuteOptions{CaptureOutput: true})
-	if err != nil {
-		t.Fatal(err)
-	}
-	gotValue, ok := result["a"]
-	if !ok {
-		t.Fatalf("expected a to be present in the result")
-	}
-	if gotValue.(string) != "b" {
-		t.Fatalf("expected a to be b, got=%v", gotValue)
-	}
-}
-
 type noopWriter struct {
 	Callback func(data []byte, level levels.Level)
 }

pkg/js/compiler/init.go

@@ -7,6 +7,7 @@ import "github.com/projectdiscovery/nuclei/v3/pkg/types"
 var (
 	// Per Execution Javascript timeout in seconds
 	JsProtocolTimeout = 10
+	JsVmConcurrency   = 500
 )
 
 // Init initializes the javascript protocol
@@ -15,6 +16,11 @@ func Init(opts *types.Options) error {
 		// keep existing 10s timeout
 		return nil
 	}
+	if opts.JsConcurrency < 100 {
+		// 100 is reasonable default
+		opts.JsConcurrency = 100
+	}
 	JsProtocolTimeout = opts.Timeout
+	JsVmConcurrency = opts.JsConcurrency
 	return nil
 }

pkg/js/compiler/pool.go

@@ -0,0 +1,116 @@
+package compiler
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/dop251/goja"
+	"github.com/dop251/goja_nodejs/console"
+	"github.com/dop251/goja_nodejs/require"
+	"github.com/projectdiscovery/gologger"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libbytes"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libfs"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libikev2"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libkerberos"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libldap"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libmssql"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libmysql"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libnet"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/liboracle"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libpop3"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libpostgres"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/librdp"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libredis"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/librsync"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libsmb"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libsmtp"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libssh"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libstructs"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libtelnet"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libvnc"
+	"github.com/projectdiscovery/nuclei/v3/pkg/js/global"
+	"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/goconsole"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
+	"github.com/remeh/sizedwaitgroup"
+)
+
+var (
+	r                *require.Registry
+	lazyRegistryInit = sync.OnceFunc(func() {
+		r = new(require.Registry) // this can be shared by multiple runtimes
+		// autoregister console node module with default printer it uses gologger backend
+		require.RegisterNativeModule(console.ModuleName, console.RequireWithPrinter(goconsole.NewGoConsolePrinter()))
+	})
+	sg         sizedwaitgroup.SizedWaitGroup
+	lazySgInit = sync.OnceFunc(func() {
+		sg = sizedwaitgroup.New(JsVmConcurrency)
+	})
+)
+
+func getRegistry() *require.Registry {
+	lazyRegistryInit()
+	return r
+}
+
+var gojapool = &sync.Pool{
+	New: func() interface{} {
+		runtime := protocolstate.NewJSRuntime()
+		_ = getRegistry().Enable(runtime)
+		// by default import below modules every time
+		_ = runtime.Set("console", require.Require(runtime, console.ModuleName))
+
+		// Register embedded javacript helpers
+		if err := global.RegisterNativeScripts(runtime); err != nil {
+			gologger.Error().Msgf("Could not register scripts: %s\n", err)
+		}
+		return runtime
+	},
+}
+
+// executes the actual js program
+func executeProgram(p *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (result goja.Value, err error) {
+	// its unknown (most likely cannot be done) to limit max js runtimes at a moment without making it static
+	// unlike sync.Pool which reacts to GC and its purposes is to reuse objects rather than creating new ones
+	lazySgInit()
+	sg.Add()
+	defer sg.Done()
+	runtime := gojapool.Get().(*goja.Runtime)
+	defer func() {
+		// reset before putting back to pool
+		_ = runtime.GlobalObject().Delete("template") // template ctx
+		// remove all args
+		for k := range args.Args {
+			_ = runtime.GlobalObject().Delete(k)
+		}
+		if opts != nil && opts.Cleanup != nil {
+			opts.Cleanup(runtime)
+		}
+		gojapool.Put(runtime)
+	}()
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("panic: %s", r)
+		}
+	}()
+	// set template ctx
+	_ = runtime.Set("template", args.TemplateCtx)
+	// set args
+	for k, v := range args.Args {
+		_ = runtime.Set(k, v)
+	}
+	// register extra callbacks if any
+	if opts != nil && opts.Callback != nil {
+		if err := opts.Callback(runtime); err != nil {
+			return nil, err
+		}
+
+	}
+	// execute the script
+	return runtime.RunProgram(p)
+}
+
+// Internal purposes i.e generating bindings
+func InternalGetGeneratorRuntime() *goja.Runtime {
+	runtime := gojapool.Get().(*goja.Runtime)
+	return runtime
+}

pkg/js/devtools/bindgen/generator.go

@@ -150,8 +150,7 @@ func CreateTemplateData(directory string, packagePrefix string) (*TemplateData,
 // InitNativeScripts initializes the native scripts array
 // with all the exported functions from the runtime
 func (d *TemplateData) InitNativeScripts() {
-	compiler := compiler.New()
-	runtime := compiler.VM()
+	runtime := compiler.InternalGetGeneratorRuntime()
 
 	exports := runtime.Get("exports")
 	if exports == nil {

pkg/protocols/http/httputils/chain.go

@@ -0,0 +1,169 @@
+package httputils
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"sync"
+
+	protoUtil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
+)
+
+// use buffer pool for storing response body
+// and reuse it for each request
+var bufPool = sync.Pool{
+	New: func() any {
+		// The Pool's New function should generally only return pointer
+		// types, since a pointer can be put into the return interface
+		// value without an allocation:
+		return new(bytes.Buffer)
+	},
+}
+
+// getBuffer returns a buffer from the pool
+func getBuffer() *bytes.Buffer {
+	return bufPool.Get().(*bytes.Buffer)
+}
+
+// putBuffer returns a buffer to the pool
+func putBuffer(buf *bytes.Buffer) {
+	buf.Reset()
+	bufPool.Put(buf)
+}
+
+// Performance Notes:
+// do not use http.Response once we create ResponseChain from it
+// as this reuses buffers and saves allocations and also drains response
+// body automatically.
+// In required cases it can be used but should never be used for anything
+// related to response body.
+// Bytes.Buffer returned by getters should not be used and are only meant for convinience
+// purposes like .String() or .Bytes() calls.
+// Remember to call Close() on ResponseChain once you are done with it.
+
+// ResponseChain is a response chain for a http request
+// on every call to previous it returns the previous response
+// if it was redirected.
+type ResponseChain struct {
+	headers      *bytes.Buffer
+	body         *bytes.Buffer
+	fullResponse *bytes.Buffer
+	resp         *http.Response
+	reloaded     bool // if response was reloaded to its previous redirect
+}
+
+// NewResponseChain creates a new response chain for a http request
+// with a maximum body size. (if -1 stick to default 4MB)
+func NewResponseChain(resp *http.Response, maxBody int64) *ResponseChain {
+	if _, ok := resp.Body.(protoUtil.LimitResponseBody); !ok {
+		resp.Body = protoUtil.NewLimitResponseBodyWithSize(resp.Body, maxBody)
+	}
+	return &ResponseChain{
+		headers:      getBuffer(),
+		body:         getBuffer(),
+		fullResponse: getBuffer(),
+		resp:         resp,
+	}
+}
+
+// Response returns the current response in the chain
+func (r *ResponseChain) Headers() *bytes.Buffer {
+	return r.headers
+}
+
+// Body returns the current response body in the chain
+func (r *ResponseChain) Body() *bytes.Buffer {
+	return r.body
+}
+
+// FullResponse returns the current response in the chain
+func (r *ResponseChain) FullResponse() *bytes.Buffer {
+	return r.fullResponse
+}
+
+// previous updates response pointer to previous response
+// if it was redirected and returns true else false
+func (r *ResponseChain) Previous() bool {
+	if r.resp != nil && r.resp.Request != nil && r.resp.Request.Response != nil {
+		r.resp = r.resp.Request.Response
+		r.reloaded = true
+		return true
+	}
+	return false
+}
+
+// Fill buffers
+func (r *ResponseChain) Fill() error {
+	r.reset()
+	if r.resp == nil {
+		return fmt.Errorf("response is nil")
+	}
+
+	// load headers
+	err := DumpResponseIntoBuffer(r.resp, false, r.headers)
+	if err != nil {
+		return fmt.Errorf("error dumping response headers: %s", err)
+	}
+
+	if r.resp.StatusCode != http.StatusSwitchingProtocols && !r.reloaded {
+		// Note about reloaded:
+		// this is a known behaviour existing from earlier version
+		// when redirect is followed and operators are executed on all redirect chain
+		// body of those requests is not available since its already been redirected
+		// This is not a issue since redirect happens with empty body according to RFC
+		// but this may be required sometimes
+		// Solution: Manual redirect using dynamic matchers or hijack redirected responses
+		// at transport level at replace with bytes buffer and then use it
+
+		// load body
+		err = readNNormalizeRespBody(r, r.body)
+		if err != nil {
+			return fmt.Errorf("error reading response body: %s", err)
+		}
+
+		// response body should not be used anymore
+		// drain and close
+		DrainResponseBody(r.resp)
+	}
+
+	// join headers and body
+	r.fullResponse.Write(r.headers.Bytes())
+	r.fullResponse.Write(r.body.Bytes())
+	return nil
+}
+
+// Close the response chain and releases the buffers.
+func (r *ResponseChain) Close() {
+	putBuffer(r.headers)
+	putBuffer(r.body)
+	putBuffer(r.fullResponse)
+	r.headers = nil
+	r.body = nil
+	r.fullResponse = nil
+}
+
+// Has returns true if the response chain has a response
+func (r *ResponseChain) Has() bool {
+	return r.resp != nil
+}
+
+// Request is request of current response
+func (r *ResponseChain) Request() *http.Request {
+	if r.resp == nil {
+		return nil
+	}
+	return r.resp.Request
+}
+
+// Response is response of current response
+func (r *ResponseChain) Response() *http.Response {
+	return r.resp
+}
+
+// reset without releasing the buffers
+// useful for redirect chain
+func (r *ResponseChain) reset() {
+	r.headers.Reset()
+	r.body.Reset()
+	r.fullResponse.Reset()
+}

pkg/protocols/http/httputils/internal.go

@@ -0,0 +1,47 @@
+package httputils
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+)
+
+// implementations copied from stdlib
+
+// errNoBody is a sentinel error value used by failureToReadBody so we
+// can detect that the lack of body was intentional.
+var errNoBody = errors.New("sentinel error value")
+
+// failureToReadBody is an io.ReadCloser that just returns errNoBody on
+// Read. It's swapped in when we don't actually want to consume
+// the body, but need a non-nil one, and want to distinguish the
+// error from reading the dummy body.
+type failureToReadBody struct{}
+
+func (failureToReadBody) Read([]byte) (int, error) { return 0, errNoBody }
+func (failureToReadBody) Close() error             { return nil }
+
+// emptyBody is an instance of empty reader.
+var emptyBody = io.NopCloser(strings.NewReader(""))
+
+// drainBody reads all of b to memory and then returns two equivalent
+// ReadClosers yielding the same bytes.
+//
+// It returns an error if the initial slurp of all bytes fails. It does not attempt
+// to make the returned ReadClosers have identical error-matching behavior.
+func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) {
+	if b == nil || b == http.NoBody {
+		// No copying needed. Preserve the magic sentinel meaning of NoBody.
+		return http.NoBody, http.NoBody, nil
+	}
+	var buf bytes.Buffer
+	if _, err = buf.ReadFrom(b); err != nil {
+		return nil, b, err
+	}
+	if err = b.Close(); err != nil {
+		return nil, b, err
+	}
+	return io.NopCloser(&buf), io.NopCloser(bytes.NewReader(buf.Bytes())), nil
+}

pkg/protocols/http/httputils/misc.go

@@ -0,0 +1,21 @@
+package httputils
+
+import (
+	"strings"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	mapsutil "github.com/projectdiscovery/utils/maps"
+)
+
+// if template contains more than 1 request and matchers require requestcondition from
+// both requests , then we need to request for event from interactsh even if current request
+// doesnot use interactsh url in it
+func GetInteractshURLSFromEvent(event map[string]interface{}) []string {
+	interactshUrls := map[string]struct{}{}
+	for k, v := range event {
+		if strings.HasPrefix(k, "interactsh-url") {
+			interactshUrls[types.ToString(v)] = struct{}{}
+		}
+	}
+	return mapsutil.GetKeys(interactshUrls)
+}

pkg/protocols/http/httputils/normalization.go

@@ -0,0 +1,77 @@
+package httputils
+
+import (
+	"bytes"
+	"compress/gzip"
+	"compress/zlib"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/pkg/errors"
+	"golang.org/x/text/encoding/simplifiedchinese"
+	"golang.org/x/text/transform"
+
+	stringsutil "github.com/projectdiscovery/utils/strings"
+)
+
+// readNNormalizeRespBody performs normalization on the http response object.
+// and fills body buffer with actual response body.
+func readNNormalizeRespBody(rc *ResponseChain, body *bytes.Buffer) (err error) {
+	response := rc.resp
+	// net/http doesn't automatically decompress the response body if an
+	// encoding has been specified by the user in the request so in case we have to
+	// manually do it.
+
+	origBody := rc.resp.Body
+	// wrap with decode if applicable
+	wrapped, err := wrapDecodeReader(response)
+	if err != nil {
+		wrapped = origBody
+	}
+	// read response body to buffer
+	_, err = body.ReadFrom(wrapped)
+	if err != nil {
+		if strings.Contains(err.Error(), "gzip: invalid header") {
+			// its invalid gzip but we will still use it from original body
+			_, err = body.ReadFrom(origBody)
+			if err != nil {
+				return errors.Wrap(err, "could not read response body after gzip error")
+			}
+		}
+		if stringsutil.ContainsAny(err.Error(), "unexpected EOF", "read: connection reset by peer", "user canceled") {
+			// keep partial body and continue (skip error) (add meta header in response for debugging)
+			response.Header.Set("x-nuclei-ignore-error", err.Error())
+			return nil
+		}
+		return errors.Wrap(err, "could not read response body")
+	}
+	return nil
+}
+
+// wrapDecodeReader wraps a decompression reader around the response body if it's compressed
+// using gzip or deflate.
+func wrapDecodeReader(resp *http.Response) (rc io.ReadCloser, err error) {
+	switch resp.Header.Get("Content-Encoding") {
+	case "gzip":
+		rc, err = gzip.NewReader(resp.Body)
+	case "deflate":
+		rc, err = zlib.NewReader(resp.Body)
+	default:
+		rc = resp.Body
+	}
+	if err != nil {
+		return nil, err
+	}
+	// handle GBK encoding
+	if isContentTypeGbk(resp.Header.Get("Content-Type")) {
+		rc = io.NopCloser(transform.NewReader(rc, simplifiedchinese.GBK.NewDecoder()))
+	}
+	return rc, nil
+}
+
+// isContentTypeGbk checks if the content-type header is gbk
+func isContentTypeGbk(contentType string) bool {
+	contentType = strings.ToLower(contentType)
+	return stringsutil.ContainsAny(contentType, "gbk", "gb2312", "gb18030")
+}

pkg/protocols/http/httputils/response.go

@@ -0,0 +1,52 @@
+package httputils
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+
+	protocolutil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
+)
+
+// DumpResponseIntoBuffer dumps a http response without allocating a new buffer
+// for the response body.
+func DumpResponseIntoBuffer(resp *http.Response, body bool, buff *bytes.Buffer) (err error) {
+	if resp == nil {
+		return fmt.Errorf("response is nil")
+	}
+	save := resp.Body
+	savecl := resp.ContentLength
+
+	if !body {
+		// For content length of zero. Make sure the body is an empty
+		// reader, instead of returning error through failureToReadBody{}.
+		if resp.ContentLength == 0 {
+			resp.Body = emptyBody
+		} else {
+			resp.Body = failureToReadBody{}
+		}
+	} else if resp.Body == nil {
+		resp.Body = emptyBody
+	} else {
+		save, resp.Body, err = drainBody(resp.Body)
+		if err != nil {
+			return err
+		}
+	}
+	err = resp.Write(buff)
+	if err == errNoBody {
+		err = nil
+	}
+	resp.Body = save
+	resp.ContentLength = savecl
+	return
+}
+
+// DrainResponseBody drains the response body and closes it.
+func DrainResponseBody(resp *http.Response) {
+	defer resp.Body.Close()
+	// don't reuse connection and just close if body length is more than 2 * MaxBodyRead
+	// to avoid DOS
+	_, _ = io.CopyN(io.Discard, resp.Body, 2*protocolutil.MaxBodyRead)
+}

pkg/protocols/http/request.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"net/http/httputil"
 	"strconv"
 	"strings"
 	"sync"
@@ -32,12 +31,14 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/tostring"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httputils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool"
 	protocolutil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/rawhttp"
+	convUtil "github.com/projectdiscovery/utils/conversion"
 	"github.com/projectdiscovery/utils/reader"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
@@ -400,7 +401,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 						MatchFunc:      request.Match,
 						ExtractFunc:    request.Extract,
 					}
-					allOASTUrls := getInteractshURLsFromEvent(event.InternalEvent)
+					allOASTUrls := httputils.GetInteractshURLSFromEvent(event.InternalEvent)
 					allOASTUrls = append(allOASTUrls, generatedHttpRequest.interactshURLs...)
 					request.options.Interactsh.RequestEvent(sliceutil.Dedupe(allOASTUrls), requestData)
 					gotMatches = request.options.Interactsh.AlreadyMatched(requestData)
@@ -685,12 +686,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		callback(event)
 		return err
 	}
-	defer func() {
-		if resp.StatusCode != http.StatusSwitchingProtocols {
-			_, _ = io.CopyN(io.Discard, resp.Body, drainReqSize)
-		}
-		resp.Body.Close()
-	}()
 
 	var curlCommand string
 	if !request.Unsafe && resp != nil && generatedRequest.request != nil && resp.Request != nil && !request.Race {
@@ -706,55 +701,39 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 	request.options.Output.Request(request.options.TemplatePath, formedURL, request.Type().String(), err)
 
 	duration := time.Since(timeStart)
-
-	dumpedResponseHeaders, err := httputil.DumpResponse(resp, false)
-	if err != nil {
-		return errors.Wrap(err, "could not dump http response")
+	// define max body read limit
+	maxBodylimit := -1 // stick to default 4MB
+	if request.MaxSize > 0 {
+		maxBodylimit = request.MaxSize
+	} else if request.options.Options.ResponseReadSize != 0 {
+		maxBodylimit = request.options.Options.ResponseReadSize
 	}
 
-	var dumpedResponse []redirectedResponse
-	var gotData []byte
-	// If the status code is HTTP 101, we should not proceed with reading body.
-	if resp.StatusCode != http.StatusSwitchingProtocols {
-		var bodyReader io.Reader
-		if request.MaxSize != 0 {
-			bodyReader = io.LimitReader(resp.Body, int64(request.MaxSize))
-		} else if request.options.Options.ResponseReadSize != 0 {
-			bodyReader = io.LimitReader(resp.Body, int64(request.options.Options.ResponseReadSize))
-		} else {
-			bodyReader = resp.Body
-		}
-		data, err := io.ReadAll(bodyReader)
-		if err != nil {
-			// Ignore body read due to server misconfiguration errors
-			if stringsutil.ContainsAny(err.Error(), "gzip: invalid header") {
-				gologger.Warning().Msgf("[%s] Server sent an invalid gzip header and it was not possible to read the uncompressed body for %s: %s", request.options.TemplateID, formedURL, err.Error())
-			} else if !stringsutil.ContainsAny(err.Error(), "unexpected EOF", "user canceled") { // ignore EOF and random error
-				return errors.Wrap(err, "could not read http body")
+	// respChain is http response chain that reads response body
+	// efficiently by reusing buffers and does all decoding and optimizations
+	respChain := httputils.NewResponseChain(resp, int64(maxBodylimit))
+	defer respChain.Close() // reuse buffers
+
+	// we only intend to log/save the final redirected response
+	// i.e why we have to use sync.Once to ensure it's only done once
+	var errx error
+	onceFunc := sync.OnceFunc(func() {
+		// if nuclei-project is enabled store the response if not previously done
+		if request.options.ProjectFile != nil && !fromCache {
+			if err := request.options.ProjectFile.Set(dumpedRequest, resp, respChain.Body().Bytes()); err != nil {
+				errx = errors.Wrap(err, "could not store in project file")
 			}
 		}
-		gotData = data
-		resp.Body.Close()
+	})
 
-		dumpedResponse, err = dumpResponseWithRedirectChain(resp, data)
-		if err != nil {
-			return errors.Wrap(err, "could not read http response with redirect chain")
-		}
-	} else {
-		dumpedResponse = []redirectedResponse{{resp: resp, fullResponse: dumpedResponseHeaders, headers: dumpedResponseHeaders}}
-	}
-
-	// if nuclei-project is enabled store the response if not previously done
-	if request.options.ProjectFile != nil && !fromCache {
-		if err := request.options.ProjectFile.Set(dumpedRequest, resp, gotData); err != nil {
-			return errors.Wrap(err, "could not store in project file")
-		}
-	}
-
-	for _, response := range dumpedResponse {
-		if response.resp == nil {
-			continue // Skip nil responses
+	// evaluate responses continiously until first redirect request in reverse order
+	for respChain.Has() {
+		// fill buffers, read response body and reuse connection
+		if err := respChain.Fill(); err != nil {
+			return errors.Wrap(err, "could not generate response chain")
 		}
+		// save response to projectfile
+		onceFunc()
 		matchedURL := input.MetaInput.Input
 		if generatedRequest.rawRequest != nil {
 			if generatedRequest.rawRequest.FullURL != "" {
@@ -767,14 +746,14 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			matchedURL = generatedRequest.request.URL.String()
 		}
 		// Give precedence to the final URL from response
-		if response.resp.Request != nil {
-			if responseURL := response.resp.Request.URL.String(); responseURL != "" {
+		if respChain.Request() != nil {
+			if responseURL := respChain.Request().URL.String(); responseURL != "" {
 				matchedURL = responseURL
 			}
 		}
 		finalEvent := make(output.InternalEvent)
 
-		outputEvent := request.responseToDSLMap(response.resp, input.MetaInput.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta)
+		outputEvent := request.responseToDSLMap(respChain.Response(), input.MetaInput.Input, matchedURL, convUtil.String(dumpedRequest), respChain.FullResponse().String(), respChain.Body().String(), respChain.Headers().String(), duration, generatedRequest.meta)
 		// add response fields to template context and merge templatectx variables to output event
 		request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent)
 		if request.options.HasTemplateCtx(input.MetaInput) {
@@ -819,9 +798,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			event.UsesInteractsh = true
 		}
 
-		responseContentType := resp.Header.Get("Content-Type")
-		isResponseTruncated := request.MaxSize > 0 && len(gotData) >= request.MaxSize
-		dumpResponse(event, request, response.fullResponse, formedURL, responseContentType, isResponseTruncated, input.MetaInput.Input)
+		responseContentType := respChain.Response().Header.Get("Content-Type")
+		isResponseTruncated := request.MaxSize > 0 && respChain.Body().Len() >= request.MaxSize
+		dumpResponse(event, request, respChain.FullResponse().Bytes(), formedURL, responseContentType, isResponseTruncated, input.MetaInput.Input)
 
 		callback(event)
 
@@ -829,8 +808,15 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		if (request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch || request.StopAtFirstMatch) && event.HasResults() {
 			return nil
 		}
+		// proceed with previous response
+		// we evaluate operators recursively for each response
+		// until we reach the first redirect response
+		if !respChain.Previous() {
+			break
+		}
 	}
-	return nil
+	// return project file save error if any
+	return errx
 }
 
 // handleSignature of the http request

pkg/protocols/http/utils.go

@@ -1,119 +1,13 @@
 package http
 
 import (
-	"bytes"
-	"compress/gzip"
-	"compress/zlib"
 	"io"
-	"net/http"
-	"net/http/httputil"
 	"strings"
 
-	"github.com/pkg/errors"
-	"golang.org/x/text/encoding/simplifiedchinese"
-	"golang.org/x/text/transform"
-
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
-	protoUtil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/rawhttp"
-	mapsutil "github.com/projectdiscovery/utils/maps"
-	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
-type redirectedResponse struct {
-	headers      []byte
-	body         []byte
-	fullResponse []byte
-	resp         *http.Response
-}
-
-// dumpResponseWithRedirectChain dumps a http response with the
-// complete http redirect chain.
-//
-// It preserves the order in which responses were given to requests
-// and returns the data to the user for matching and viewing in that order.
-//
-// Inspired from - https://github.com/ffuf/ffuf/issues/324#issuecomment-719858923
-func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirectedResponse, error) {
-	var response []redirectedResponse
-
-	respData, err := httputil.DumpResponse(resp, false)
-	if err != nil {
-		return nil, err
-	}
-	respObj := redirectedResponse{
-		headers:      respData,
-		body:         body,
-		resp:         resp,
-		fullResponse: bytes.Join([][]byte{respData, body}, []byte{}),
-	}
-	if err := normalizeResponseBody(resp, &respObj); err != nil {
-		return nil, err
-	}
-	response = append(response, respObj)
-
-	var redirectResp *http.Response
-	if resp != nil && resp.Request != nil {
-		redirectResp = resp.Request.Response
-	}
-	for redirectResp != nil {
-		var body []byte
-
-		respData, err := httputil.DumpResponse(redirectResp, false)
-		if err != nil {
-			break
-		}
-		if redirectResp.Body != nil {
-			body, _ = protoUtil.LimitBodyRead(redirectResp.Body)
-		}
-		respObj := redirectedResponse{
-			headers:      respData,
-			body:         body,
-			resp:         redirectResp,
-			fullResponse: bytes.Join([][]byte{respData, body}, []byte{}),
-		}
-		if err := normalizeResponseBody(redirectResp, &respObj); err != nil {
-			return nil, err
-		}
-		response = append(response, respObj)
-		redirectResp = redirectResp.Request.Response
-	}
-	return response, nil
-}
-
-// normalizeResponseBody performs normalization on the http response object.
-func normalizeResponseBody(resp *http.Response, response *redirectedResponse) error {
-	var err error
-	// net/http doesn't automatically decompress the response body if an
-	// encoding has been specified by the user in the request so in case we have to
-	// manually do it.
-	dataOrig := response.body
-	response.body, err = handleDecompression(resp, response.body)
-	// in case of error use original data
-	if err != nil {
-		response.body = dataOrig
-	}
-	response.fullResponse = bytes.ReplaceAll(response.fullResponse, dataOrig, response.body)
-
-	// Decode gbk response content-types
-	// gb18030 supersedes gb2312
-	responseContentType := resp.Header.Get("Content-Type")
-	if isContentTypeGbk(responseContentType) {
-		response.fullResponse, err = decodeGBK(response.fullResponse)
-		if err != nil {
-			return errors.Wrap(err, "could not gbk decode")
-		}
-
-		// the uncompressed body needs to be decoded to standard utf8
-		response.body, err = decodeGBK(response.body)
-		if err != nil {
-			return errors.Wrap(err, "could not gbk decode")
-		}
-	}
-	return nil
-}
-
 // dump creates a dump of the http request in form of a byte slice
 func dump(req *generatedRequest, reqURL string) ([]byte, error) {
 	if req.request != nil {
@@ -122,60 +16,3 @@ func dump(req *generatedRequest, reqURL string) ([]byte, error) {
 	rawHttpOptions := &rawhttp.Options{CustomHeaders: req.rawRequest.UnsafeHeaders, CustomRawBytes: req.rawRequest.UnsafeRawBytes}
 	return rawhttp.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), io.NopCloser(strings.NewReader(req.rawRequest.Data)), rawHttpOptions)
 }
-
-// handleDecompression if the user specified a custom encoding (as golang transport doesn't do this automatically)
-func handleDecompression(resp *http.Response, bodyOrig []byte) (bodyDec []byte, err error) {
-	if resp == nil {
-		return bodyOrig, nil
-	}
-
-	var reader io.ReadCloser
-	switch resp.Header.Get("Content-Encoding") {
-	case "gzip":
-		reader, err = gzip.NewReader(bytes.NewReader(bodyOrig))
-	case "deflate":
-		reader, err = zlib.NewReader(bytes.NewReader(bodyOrig))
-	default:
-		return bodyOrig, nil
-	}
-	if err != nil {
-		return nil, err
-	}
-	defer reader.Close()
-
-	bodyDec, err = io.ReadAll(reader)
-	if err != nil {
-		return bodyOrig, err
-	}
-	return bodyDec, nil
-}
-
-// decodeGBK converts GBK to UTF-8
-func decodeGBK(s []byte) ([]byte, error) {
-	I := bytes.NewReader(s)
-	O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder())
-	d, e := io.ReadAll(O)
-	if e != nil {
-		return nil, e
-	}
-	return d, nil
-}
-
-// isContentTypeGbk checks if the content-type header is gbk
-func isContentTypeGbk(contentType string) bool {
-	contentType = strings.ToLower(contentType)
-	return stringsutil.ContainsAny(contentType, "gbk", "gb2312", "gb18030")
-}
-
-// if template contains more than 1 request and matchers require requestcondition from
-// both requests , then we need to request for event from interactsh even if current request
-// doesnot use interactsh url in it
-func getInteractshURLsFromEvent(event map[string]interface{}) []string {
-	interactshUrls := map[string]struct{}{}
-	for k, v := range event {
-		if strings.HasPrefix(k, "interactsh-url") {
-			interactshUrls[types.ToString(v)] = struct{}{}
-		}
-	}
-	return mapsutil.GetKeys(interactshUrls)
-}

pkg/protocols/javascript/js.go

@@ -91,6 +91,10 @@ type Request struct {
 
 	// cache any variables that may be needed for operation.
 	options *protocols.ExecutorOptions `yaml:"-" json:"-"`
+
+	preConditionCompiled *goja.Program `yaml:"-" json:"-"`
+
+	scriptCompiled *goja.Program `yaml:"-" json:"-"`
 }
 
 // Compile compiles the request generators preparing any requests possible.
@@ -196,13 +200,21 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 				},
 			})
 		}
+		opts.Cleanup = func(runtime *goja.Runtime) {
+			_ = runtime.GlobalObject().Delete("set")
+			_ = runtime.GlobalObject().Delete("updatePayload")
+		}
 
 		args := compiler.NewExecuteArgs()
 		allVars := generators.MergeMaps(options.Variables.GetAll(), options.Options.Vars.AsMap(), request.options.Constants)
 		// proceed with whatever args we have
 		args.Args, _ = request.evaluateArgs(allVars, options, true)
 
-		result, err := request.options.JsCompiler.ExecuteWithOptions(request.Init, args, opts)
+		initCompiled, err := goja.Compile("", request.Init, false)
+		if err != nil {
+			return errorutil.NewWithTag(request.TemplateID, "could not compile init code: %s", err)
+		}
+		result, err := request.options.JsCompiler.ExecuteWithOptions(initCompiled, args, opts)
 		if err != nil {
 			return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err)
 		}
@@ -217,6 +229,24 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 		}
 	}
 
+	// compile pre-condition if any
+	if request.PreCondition != "" {
+		preConditionCompiled, err := goja.Compile("", request.PreCondition, false)
+		if err != nil {
+			return errorutil.NewWithTag(request.TemplateID, "could not compile pre-condition: %s", err)
+		}
+		request.preConditionCompiled = preConditionCompiled
+	}
+
+	// compile actual source code
+	if request.Code != "" {
+		scriptCompiled, err := goja.Compile("", request.Code, false)
+		if err != nil {
+			return errorutil.NewWithTag(request.TemplateID, "could not compile javascript code: %s", err)
+		}
+		request.scriptCompiled = scriptCompiled
+	}
+
 	return nil
 }
 
@@ -307,7 +337,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
 		}
 		argsCopy.TemplateCtx = templateCtx.GetAll()
 
-		result, err := request.options.JsCompiler.ExecuteWithOptions(request.PreCondition, argsCopy, &compiler.ExecuteOptions{Timeout: request.Timeout})
+		result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, argsCopy, &compiler.ExecuteOptions{Timeout: request.Timeout})
 		if err != nil {
 			return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err)
 		}
@@ -425,18 +455,21 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
 		argsCopy.TemplateCtx = map[string]interface{}{}
 	}
 
-	var requestData = []byte(request.Code)
 	var interactshURLs []string
 	if request.options.Interactsh != nil {
-		var transformedData string
-		transformedData, interactshURLs = request.options.Interactsh.Replace(string(request.Code), []string{})
-		requestData = []byte(transformedData)
+		if argsCopy.Args != nil {
+			for k, v := range argsCopy.Args {
+				var urls []string
+				v, urls = request.options.Interactsh.Replace(fmt.Sprint(v), []string{})
+				if len(urls) > 0 {
+					interactshURLs = append(interactshURLs, urls...)
+					argsCopy.Args[k] = v
+				}
+			}
+		}
 	}
 
-	results, err := request.options.JsCompiler.ExecuteWithOptions(string(requestData), argsCopy, &compiler.ExecuteOptions{
-		Pool:    false,
-		Timeout: request.Timeout,
-	})
+	results, err := request.options.JsCompiler.ExecuteWithOptions(request.scriptCompiled, argsCopy, &compiler.ExecuteOptions{Timeout: request.Timeout})
 	if err != nil {
 		// shouldn't fail even if it returned error instead create a failure event
 		results = compiler.ExecuteResult{"success": false, "error": err.Error()}

pkg/protocols/utils/reader.go

@@ -18,11 +18,21 @@ type LimitResponseBody struct {
 // NewLimitResponseBody wraps response body with a limit reader.
 // thus only allowing MaxBodyRead bytes to be read. i.e 4MB
 func NewLimitResponseBody(body io.ReadCloser) io.ReadCloser {
+	return NewLimitResponseBodyWithSize(body, MaxBodyRead)
+}
+
+// NewLimitResponseBody wraps response body with a limit reader.
+// thus only allowing MaxBodyRead bytes to be read. i.e 4MB
+func NewLimitResponseBodyWithSize(body io.ReadCloser, size int64) io.ReadCloser {
 	if body == nil {
 		return nil
 	}
+	if size == -1 {
+		// stick to default 4MB
+		size = MaxBodyRead
+	}
 	return &LimitResponseBody{
-		Reader: io.LimitReader(body, MaxBodyRead),
+		Reader: io.LimitReader(body, size),
 		Closer: body,
 	}
 }

pkg/tmplexec/exec.go

@@ -6,11 +6,11 @@ import (
 	"strings"
 	"sync/atomic"
 
+	"github.com/dop251/goja"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
 	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow"
@@ -24,6 +24,7 @@ type TemplateExecuter struct {
 	options  *protocols.ExecutorOptions
 	engine   TemplateEngine
 	results  *atomic.Bool
+	program  *goja.Program
 }
 
 // Both executer & Executor are correct spellings (its open to interpretation)
@@ -47,11 +48,11 @@ func NewTemplateExecuter(requests []protocols.Request, options *protocols.Execut
 		// we use a dummy input here because goal of flow executor at this point is to just check
 		// syntax and other things are correct before proceeding to actual execution
 		// during execution new instance of flow will be created as it is tightly coupled with lot of executor options
-		var err error
-		e.engine, err = flow.NewFlowExecutor(requests, scan.NewScanContext(contextargs.NewWithInput("dummy")), options, e.results)
+		p, err := goja.Compile("flow.js", options.Flow, false)
 		if err != nil {
-			return nil, fmt.Errorf("could not create flow executor: %s", err)
+			return nil, fmt.Errorf("could not compile flow: %s", err)
 		}
+		e.program = p
 	} else {
 		// Review:
 		// multiproto engine is only used if there is more than one protocol in template
@@ -84,6 +85,10 @@ func (e *TemplateExecuter) Compile() error {
 			return err
 		}
 	}
+	if e.engine == nil && e.options.Flow != "" {
+		// this is true for flow executor
+		return nil
+	}
 	return e.engine.Compile()
 }
 
@@ -158,7 +163,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
 	// so in compile step earlier we compile it to validate javascript syntax and other things
 	// and while executing we create new instance of flow executor everytime
 	if e.options.Flow != "" {
-		flowexec, err := flow.NewFlowExecutor(e.requests, ctx, e.options, results)
+		flowexec, err := flow.NewFlowExecutor(e.requests, ctx, e.options, results, e.program)
 		if err != nil {
 			ctx.LogError(err)
 			return false, fmt.Errorf("could not create flow executor: %s", err)

pkg/tmplexec/flow/flow_executor.go

@@ -11,7 +11,6 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 
@@ -40,12 +39,11 @@ type FlowExecutor struct {
 	options *protocols.ExecutorOptions
 
 	// javascript runtime reference and compiled program
-	jsVM    *goja.Runtime
 	program *goja.Program // compiled js program
 
 	// protocol requests and their callback functions
 	allProtocols   map[string][]protocols.Request
-	protoFunctions map[string]func(call goja.FunctionCall) goja.Value // reqFunctions contains functions that allow executing requests/protocols from js
+	protoFunctions map[string]func(call goja.FunctionCall, runtime *goja.Runtime) goja.Value // reqFunctions contains functions that allow executing requests/protocols from js
 
 	// logic related variables
 	results *atomic.Bool
@@ -58,7 +56,7 @@ type FlowExecutor struct {
 // NewFlowExecutor creates a new flow executor from a list of requests
 // Note: Unlike other engine for every target x template flow needs to be compiled and executed everytime
 // unlike other engines where we compile once and execute multiple times
-func NewFlowExecutor(requests []protocols.Request, ctx *scan.ScanContext, options *protocols.ExecutorOptions, results *atomic.Bool) (*FlowExecutor, error) {
+func NewFlowExecutor(requests []protocols.Request, ctx *scan.ScanContext, options *protocols.ExecutorOptions, results *atomic.Bool, program *goja.Program) (*FlowExecutor, error) {
 	allprotos := make(map[string][]protocols.Request)
 	for _, req := range requests {
 		switch req.Type() {
@@ -96,10 +94,10 @@ func NewFlowExecutor(requests []protocols.Request, ctx *scan.ScanContext, option
 			ReadOnly: atomic.Bool{},
 			Map:      make(map[string]error),
 		},
-		protoFunctions: map[string]func(call goja.FunctionCall) goja.Value{},
+		protoFunctions: map[string]func(call goja.FunctionCall, runtime *goja.Runtime) goja.Value{},
 		results:        results,
-		jsVM:           protocolstate.NewJSRuntime(),
 		ctx:            ctx,
+		program:        program,
 	}
 	return f, nil
 }
@@ -130,7 +128,7 @@ func (f *FlowExecutor) Compile() error {
 	f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Merge(allVars) // merge all variables into template context
 
 	// ---- define callback functions/objects----
-	f.protoFunctions = map[string]func(call goja.FunctionCall) goja.Value{}
+	f.protoFunctions = map[string]func(call goja.FunctionCall, runtime *goja.Runtime) goja.Value{}
 	// iterate over all protocols and generate callback functions for each protocol
 	for p, requests := range f.allProtocols {
 		// for each protocol build a requestMap with reqID and protocol request
@@ -150,7 +148,7 @@ func (f *FlowExecutor) Compile() error {
 		}
 		// ---define hook that allows protocol/request execution from js-----
 		// --- this is the actual callback that is executed when function is invoked in js----
-		f.protoFunctions[proto] = func(call goja.FunctionCall) goja.Value {
+		f.protoFunctions[proto] = func(call goja.FunctionCall, runtime *goja.Runtime) goja.Value {
 			opts := &ProtoOptions{
 				protoName: proto,
 			}
@@ -169,10 +167,10 @@ func (f *FlowExecutor) Compile() error {
 					}
 				}
 			}
-			return f.jsVM.ToValue(f.requestExecutor(reqMap, opts))
+			return runtime.ToValue(f.requestExecutor(runtime, reqMap, opts))
 		}
 	}
-	return f.registerBuiltInFunctions()
+	return nil
 }
 
 // ExecuteWithResults executes the flow and returns results
@@ -192,11 +190,50 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error {
 			f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(key, value)
 		})
 	}
+
+	// get a new runtime from pool
+	runtime := GetJSRuntime(f.options.Options)
+	defer PutJSRuntime(runtime) // put runtime back to pool
+	defer func() {
+		// remove set builtin
+		_ = runtime.GlobalObject().Delete("set")
+		_ = runtime.GlobalObject().Delete("template")
+		for proto := range f.protoFunctions {
+			_ = runtime.GlobalObject().Delete(proto)
+		}
+
+	}()
+	defer func() {
+		if r := recover(); r != nil {
+			f.ctx.LogError(fmt.Errorf("panic occurred while executing flow: %v", r))
+		}
+	}()
+
 	if ctx.OnResult == nil {
 		return fmt.Errorf("output callback cannot be nil")
 	}
+	// before running register set of builtins
+	if err := runtime.Set("set", func(call goja.FunctionCall) goja.Value {
+		varName := call.Argument(0).Export()
+		varValue := call.Argument(1).Export()
+		f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(types.ToString(varName), varValue)
+		return goja.Null()
+	}); err != nil {
+		return err
+	}
+	// also register functions that allow executing protocols from js
+	for proto, fn := range f.protoFunctions {
+		if err := runtime.Set(proto, fn); err != nil {
+			return err
+		}
+	}
+	// register template object
+	if err := runtime.Set("template", f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()); err != nil {
+		return err
+	}
+
 	// pass flow and execute the js vm and handle errors
-	_, err := f.jsVM.RunProgram(f.program)
+	_, err := runtime.RunProgram(f.program)
 	if err != nil {
 		ctx.LogError(err)
 		return errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow)

pkg/tmplexec/flow/flow_internal.go

@@ -2,24 +2,18 @@ package flow
 
 import (
 	"fmt"
-	"reflect"
 	"sync/atomic"
 
 	"github.com/dop251/goja"
-	"github.com/logrusorgru/aurora"
-	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
-	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow/builtin"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 )
 
 // contains all internal/unexported methods of flow
 
 // requestExecutor executes a protocol/request and returns true if any matcher was found
-func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Request], opts *ProtoOptions) bool {
+func (f *FlowExecutor) requestExecutor(runtime *goja.Runtime, reqMap mapsutil.Map[string, protocols.Request], opts *ProtoOptions) bool {
 	defer func() {
 		// evaluate all variables after execution of each protocol
 		variableMap := f.options.Variables.Evaluate(f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll())
@@ -27,7 +21,7 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req
 
 		// to avoid polling update template variables everytime we execute a protocol
 		var m map[string]interface{} = f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()
-		_ = f.jsVM.Set("template", m)
+		_ = runtime.Set("template", m)
 	}()
 	matcherStatus := &atomic.Bool{} // due to interactsh matcher polling logic this needs to be atomic bool
 	// if no id is passed execute all requests in sequence
@@ -117,99 +111,3 @@ func (f *FlowExecutor) protocolResultCallback(req protocols.Request, matcherStat
 		}
 	}
 }
-
-// registerBuiltInFunctions registers all built in functions for the flow
-func (f *FlowExecutor) registerBuiltInFunctions() error {
-	// currently we register following builtin functions
-	// log -> log to stdout with [JS] prefix should only be used for debugging
-	// set -> set a variable in template context
-	// proto(arg ...String) <- this is generic syntax of how a protocol/request binding looks in js
-	// we only register only those protocols that are available in template
-
-	// we also register a map datatype called template with all template variables
-	// template -> all template variables are available in js template object
-
-	if err := f.jsVM.Set("log", func(call goja.FunctionCall) goja.Value {
-		// TODO: verify string interpolation and handle multiple args
-		arg := call.Argument(0).Export()
-		switch value := arg.(type) {
-		case string:
-			gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value)
-		case map[string]interface{}:
-			gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), vardump.DumpVariables(value))
-		default:
-			gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value)
-		}
-		return call.Argument(0) // return the same value
-	}); err != nil {
-		return err
-	}
-
-	if err := f.jsVM.Set("set", func(call goja.FunctionCall) goja.Value {
-		varName := call.Argument(0).Export()
-		varValue := call.Argument(1).Export()
-		f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(types.ToString(varName), varValue)
-		return goja.Null()
-	}); err != nil {
-		return err
-	}
-
-	// iterate provides global iterator function by handling null values or strings
-	if err := f.jsVM.Set("iterate", func(call goja.FunctionCall) goja.Value {
-		allVars := []any{}
-		for _, v := range call.Arguments {
-			if v.Export() == nil {
-				continue
-			}
-			if v.ExportType().Kind() == reflect.Slice {
-				// convert []datatype to []interface{}
-				// since it cannot be type asserted to []interface{} directly
-				rfValue := reflect.ValueOf(v.Export())
-				for i := 0; i < rfValue.Len(); i++ {
-					allVars = append(allVars, rfValue.Index(i).Interface())
-				}
-			} else {
-				allVars = append(allVars, v.Export())
-			}
-		}
-		return f.jsVM.ToValue(allVars)
-	}); err != nil {
-		return err
-	}
-
-	// add a builtin dedupe object
-	if err := f.jsVM.Set("Dedupe", func(call goja.ConstructorCall) *goja.Object {
-		d := builtin.NewDedupe(f.jsVM)
-		obj := call.This
-		// register these methods
-		_ = obj.Set("Add", d.Add)
-		_ = obj.Set("Values", d.Values)
-		return nil
-	}); err != nil {
-		return err
-	}
-
-	var m = f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()
-	if m == nil {
-		m = map[string]interface{}{}
-	}
-
-	if err := f.jsVM.Set("template", m); err != nil {
-		// all template variables are available in js template object
-		return err
-	}
-
-	// register all protocols
-	for name, fn := range f.protoFunctions {
-		if err := f.jsVM.Set(name, fn); err != nil {
-			return err
-		}
-	}
-
-	program, err := goja.Compile("flow", f.options.Flow, false)
-	if err != nil {
-		return err
-	}
-	f.program = program
-	return nil
-}

pkg/tmplexec/flow/vm.go

@@ -0,0 +1,94 @@
+package flow
+
+import (
+	"reflect"
+	"sync"
+
+	"github.com/dop251/goja"
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
+	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow/builtin"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/remeh/sizedwaitgroup"
+)
+
+type jsWaitGroup struct {
+	sync.Once
+	sg sizedwaitgroup.SizedWaitGroup
+}
+
+var jsPool = &jsWaitGroup{}
+
+// GetJSRuntime returns a new JS runtime from pool
+func GetJSRuntime(opts *types.Options) *goja.Runtime {
+	jsPool.Do(func() {
+		if opts.JsConcurrency < 100 {
+			opts.JsConcurrency = 100
+		}
+		jsPool.sg = sizedwaitgroup.New(opts.JsConcurrency)
+	})
+	jsPool.sg.Add()
+	return gojapool.Get().(*goja.Runtime)
+}
+
+// PutJSRuntime returns a JS runtime to pool
+func PutJSRuntime(runtime *goja.Runtime) {
+	defer jsPool.sg.Done()
+	gojapool.Put(runtime)
+}
+
+// js runtime pool using sync.Pool
+var gojapool = &sync.Pool{
+	New: func() interface{} {
+		runtime := protocolstate.NewJSRuntime()
+		registerBuiltins(runtime)
+		return runtime
+	},
+}
+
+func registerBuiltins(runtime *goja.Runtime) {
+	_ = runtime.Set("log", func(call goja.FunctionCall) goja.Value {
+		// TODO: verify string interpolation and handle multiple args
+		arg := call.Argument(0).Export()
+		switch value := arg.(type) {
+		case string:
+			gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value)
+		case map[string]interface{}:
+			gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), vardump.DumpVariables(value))
+		default:
+			gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value)
+		}
+		return call.Argument(0) // return the same value
+	})
+
+	_ = runtime.Set("iterate", func(call goja.FunctionCall) goja.Value {
+		allVars := []any{}
+		for _, v := range call.Arguments {
+			if v.Export() == nil {
+				continue
+			}
+			if v.ExportType().Kind() == reflect.Slice {
+				// convert []datatype to []interface{}
+				// since it cannot be type asserted to []interface{} directly
+				rfValue := reflect.ValueOf(v.Export())
+				for i := 0; i < rfValue.Len(); i++ {
+					allVars = append(allVars, rfValue.Index(i).Interface())
+				}
+			} else {
+				allVars = append(allVars, v.Export())
+			}
+		}
+		return runtime.ToValue(allVars)
+	})
+
+	_ = runtime.Set("Dedupe", func(call goja.ConstructorCall) *goja.Object {
+		d := builtin.NewDedupe(runtime)
+		obj := call.This
+		// register these methods
+		_ = obj.Set("Add", d.Add)
+		_ = obj.Set("Values", d.Values)
+		return nil
+	})
+}

pkg/types/types.go

@@ -366,6 +366,8 @@ type Options struct {
 	EnableCloudUpload bool
 	// ScanID is the scan ID to use for cloud upload
 	ScanID string
+	// JsConcurrency is the number of concurrent js routines to run
+	JsConcurrency int
 }
 
 // ShouldLoadResume resume file