internal sync fix + speed up

examples/with_speed_control/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"log"
 	"sync"
 	"time"
 
@@ -9,11 +10,32 @@ import (
 )
 
 func main() {
-	ne, err := nuclei.NewNucleiEngine(
+	ne, err := initializeNucleiEngine()
-		nuclei.WithTemplateFilters(nuclei.TemplateFilters{
+	if err != nil {
-			IDs:         []string{"header-command-injection"},
+		panic(err)
-			IncludeTags: []string{"fuzz"},
+	}
-		}),
+	defer ne.Close()
+
+	ne.LoadTargets([]string{"http://honey.scanme.sh"}, false)
+
+	var wg sync.WaitGroup
+	wg.Add(3)
+
+	go testRateLimit(&wg, ne)
+	go testThreadsAndBulkSize(&wg, ne)
+	go testPayloadConcurrency(&wg, ne)
+
+	err = ne.ExecuteWithCallback(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	wg.Wait()
+}
+
+func initializeNucleiEngine() (*nuclei.NucleiEngine, error) {
+	return nuclei.NewNucleiEngine(
+		nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "http"}),
 		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}),
 		nuclei.WithGlobalRateLimit(1, time.Second),
 		nuclei.WithConcurrency(nuclei.Concurrency{
@@ -25,78 +47,57 @@ func main() {
 			TemplatePayloadConcurrency:    1,
 		}),
 	)
-	if err != nil {
+}
-		panic(err)
-	}
-	// load targets and optionally probe non http/https targets
-	ne.LoadTargets([]string{"http://honey.scanme.sh"}, false)
 
-	var wgtest sync.WaitGroup
+func testRateLimit(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
+	defer wg.Done()
+	verifyRateLimit(ne, 1, 5000)
+}
 
-	// speed tests
+func testThreadsAndBulkSize(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
-	// increase rate limit
+	defer wg.Done()
-	wgtest.Add(1)
+	initialTemplateThreads, initialBulkSize := 1, 1
-	go func() {
+	verifyThreadsAndBulkSize(ne, initialTemplateThreads, initialBulkSize, 25, 25)
-		defer wgtest.Done()
+}
-		initialRate := ne.GetExecuterOptions().RateLimiter.GetLimit()
+
-		if initialRate != 1 {
+func testPayloadConcurrency(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
+	defer wg.Done()
+	verifyPayloadConcurrency(ne, 1, 500)
+}
+
+func verifyRateLimit(ne *nuclei.NucleiEngine, initialRate, finalRate int) {
+	if ne.GetExecuterOptions().RateLimiter.GetLimit() != uint(initialRate) {
 		panic("wrong initial rate limit")
 	}
-		time.Sleep(10 * time.Second)
+	time.Sleep(5 * time.Second)
-		ne.Options().RateLimit = 1000
+	ne.Options().RateLimit = finalRate
-		time.Sleep(10 * time.Second)
+	time.Sleep(20 * time.Second)
-		finalRate := ne.GetExecuterOptions().RateLimiter.GetLimit()
+	if ne.GetExecuterOptions().RateLimiter.GetLimit() != uint(finalRate) {
-		if finalRate != 1000 {
 		panic("wrong final rate limit")
 	}
-	}()
+}
 
-	// increase threads and bulk size
+func verifyThreadsAndBulkSize(ne *nuclei.NucleiEngine, initialThreads, initialBulk, finalThreads, finalBulk int) {
-	wgtest.Add(1)
+	if ne.Options().TemplateThreads != initialThreads || ne.Options().BulkSize != initialBulk {
-	go func() {
-		defer wgtest.Done()
-		initialTemplateThreads := ne.Options().TemplateThreads
-		initialBulkSize := ne.Options().BulkSize
-		if initialTemplateThreads != 1 || initialBulkSize != 1 {
 		panic("wrong initial standard concurrency")
 	}
-		time.Sleep(10 * time.Second)
+	time.Sleep(5 * time.Second)
-		ne.Options().TemplateThreads = 5
+	ne.Options().TemplateThreads = finalThreads
-		ne.Options().BulkSize = 25
+	ne.Options().BulkSize = finalBulk
-		time.Sleep(10 * time.Second)
+	time.Sleep(20 * time.Second)
-		// check new values via workpool
+	if ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size != finalBulk || ne.Engine().WorkPool().Default.Size != finalThreads {
-		finalTemplateThreads := ne.Engine().WorkPool().Default.Size
+		log.Fatal("wrong final concurrency", ne.Engine().WorkPool().Default.Size, finalThreads, ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size, finalBulk)
-		finalBulkSize := ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size
+	}
-		if finalTemplateThreads != 5 && finalBulkSize != 25 {
+}
-			panic("wrong final concurreny")
+
+func verifyPayloadConcurrency(ne *nuclei.NucleiEngine, initialPayloadConcurrency, finalPayloadConcurrency int) {
+	if ne.Options().PayloadConcurrency != initialPayloadConcurrency {
+		panic("wrong initial payload concurrency")
+	}
+	time.Sleep(5 * time.Second)
+	ne.Options().PayloadConcurrency = finalPayloadConcurrency
+	time.Sleep(20 * time.Second)
+	if ne.GetExecuterOptions().GetThreadsForNPayloadRequests(100, 0) != finalPayloadConcurrency {
+		panic("wrong final payload concurrency")
 	}
-	}()
-
-	// increase payload concurrency
-	wgtest.Add(1)
-	go func() {
-		defer wgtest.Done()
-		initialpayloadConcurrency := ne.Options().PayloadConcurrency
-		if initialpayloadConcurrency != 1 {
-			panic("wrong initial payload concurrency")
-		}
-		time.Sleep(10 * time.Second)
-		ne.Options().PayloadConcurrency = 100
-		time.Sleep(10 * time.Second)
-
-		// the ongoing and next payload iterations will retrieve parallelism from this function
-		// it should have the new set value, that will be cascade applied to all running adaptive wait groups
-		finalPayloadConcurrency := ne.GetExecuterOptions().GetThreadsForNPayloadRequests(100, 0)
-		if finalPayloadConcurrency != 100 {
-			panic("wrong initial payload concurrency")
-		}
-	}()
-
-	err = ne.ExecuteWithCallback(nil)
-	if err != nil {
-		panic(err)
-	}
-	defer ne.Close()
-
-	wgtest.Wait()
 }

go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222
 	github.com/xanzy/go-gitlab v0.84.0
 	go.uber.org/multierr v1.11.0
-	golang.org/x/net v0.21.0
+	golang.org/x/net v0.24.0
 	golang.org/x/oauth2 v0.11.0
 	golang.org/x/text v0.14.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -100,7 +100,7 @@ require (
 	github.com/seh-msft/burpxml v1.0.1
 	github.com/stretchr/testify v1.9.0
 	github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
-	golang.org/x/term v0.17.0
+	golang.org/x/term v0.19.0
 	gopkg.in/yaml.v3 v3.0.1
 	moul.io/http2curl v1.0.0
 )
@@ -302,10 +302,10 @@ require (
 	go.etcd.io/bbolt v1.3.8 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	goftp.io/server/v2 v2.0.1 // indirect
-	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
 	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.17.0
 	google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -1186,8 +1186,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
 golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1280,8 +1280,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1382,8 +1382,9 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1395,8 +1396,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
-golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
+golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

pkg/core/engine.go

@@ -58,5 +58,7 @@ func (e *Engine) ExecuterOptions() protocols.ExecutorOptions {
 
 // WorkPool returns the worker pool for the engine
 func (e *Engine) WorkPool() *WorkPool {
+	// resize check point - nop if there are no changes
+	e.workPool.RefreshWithConfig(e.GetWorkPoolConfig())
 	return e.workPool
 }

pkg/core/executors.go

@@ -217,7 +217,7 @@ func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs
 	templateType := template.Type()
 
 	// resize check point - nop if there are no changes
-	e.e.workPool.RefreshWithConfig(e.e.GetWorkPoolConfig())
+	e.e.WorkPool().RefreshWithConfig(e.e.GetWorkPoolConfig())
 
 	var wg *syncutil.AdaptiveWaitGroup
 	if templateType == types.HeadlessProtocol {