diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 7ebdaee1..aa0d8aec 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -66,3 +66,7 @@ jobs: - name: Example SDK Advanced run: go run . working-directory: examples/advanced/ + + - name: Example SDK with speed control + run: go run . + working-directory: examples/with_speed_control/ diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 3caec21c..a90c4ed5 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -15,6 +15,7 @@ import ( "github.com/projectdiscovery/utils/auth/pdcp" "github.com/projectdiscovery/utils/env" _ "github.com/projectdiscovery/utils/pprof" + stringsutil "github.com/projectdiscovery/utils/strings" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" @@ -329,13 +330,15 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.CreateGroup("rate-limit", "Rate-Limit", flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"), - flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"), + flagSet.DurationVarP(&options.RateLimitDuration, "rate-limit-duration", "rld", time.Second, "maximum number of requests to send per second"), + flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute (DEPRECATED)"), flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"), flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"), flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"), flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "headc", 10, "maximum number of headless templates to be executed in parallel"), flagSet.IntVarP(&options.JsConcurrency, "js-concurrency", "jsc", 120, "maximum number of javascript runtimes to be executed in parallel"), flagSet.IntVarP(&options.PayloadConcurrency, "payload-concurrency", "pc", 25, "max payload concurrency for each template"), + flagSet.IntVarP(&options.ProbeConcurrency, "probe-concurrency", "prc", 50, "http probe concurrency with httpx"), ) flagSet.CreateGroup("optimization", "Optimizations", flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait in seconds before timeout"), @@ -597,10 +600,10 @@ Note: Make sure you have backup of your custom nuclei-templates before proceedin gologger.Fatal().Msgf("could not read response: %s", err) } resp = strings.TrimSpace(resp) - if strings.EqualFold(resp, "y") || strings.EqualFold(resp, "yes") { + if stringsutil.EqualFoldAny(resp, "y", "yes") { break } - if strings.EqualFold(resp, "n") || strings.EqualFold(resp, "no") || resp == "" { + if stringsutil.EqualFoldAny(resp, "n", "no", "") { fmt.Println("Exiting...") os.Exit(0) } diff --git a/examples/advanced/advanced.go b/examples/advanced/advanced.go index 5ce579b3..110160f9 100644 --- a/examples/advanced/advanced.go +++ b/examples/advanced/advanced.go @@ -2,7 +2,7 @@ package main import ( nuclei "github.com/projectdiscovery/nuclei/v3/lib" - "github.com/remeh/sizedwaitgroup" + syncutil "github.com/projectdiscovery/utils/sync" ) func main() { @@ -12,7 +12,10 @@ func main() { panic(err) } // setup sizedWaitgroup to handle concurrency - sg := sizedwaitgroup.New(10) + sg, err := syncutil.New(syncutil.WithSize(10)) + if err != nil { + panic(err) + } // scan 1 = run dns templates on scanme.sh sg.Add() diff --git a/examples/with_speed_control/main.go b/examples/with_speed_control/main.go new file mode 100644 index 00000000..b56df967 --- /dev/null +++ b/examples/with_speed_control/main.go @@ -0,0 +1,104 @@ +package main + +import ( + "log" + "sync" + "time" + + nuclei "github.com/projectdiscovery/nuclei/v3/lib" + "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" +) + +func main() { + ne, err := initializeNucleiEngine() + if err != nil { + panic(err) + } + defer ne.Close() + + ne.LoadTargets([]string{"http://honey.scanme.sh"}, false) + + var wg sync.WaitGroup + wg.Add(3) + + go testRateLimit(&wg, ne) + go testThreadsAndBulkSize(&wg, ne) + go testPayloadConcurrency(&wg, ne) + + err = ne.ExecuteWithCallback(nil) + if err != nil { + panic(err) + } + + wg.Wait() +} + +func initializeNucleiEngine() (*nuclei.NucleiEngine, error) { + return nuclei.NewNucleiEngine( + nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"oast"}}), + nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}), + nuclei.WithGlobalRateLimit(1, time.Second), + nuclei.WithConcurrency(nuclei.Concurrency{ + TemplateConcurrency: 1, + HostConcurrency: 1, + HeadlessHostConcurrency: 1, + HeadlessTemplateConcurrency: 1, + JavascriptTemplateConcurrency: 1, + TemplatePayloadConcurrency: 1, + ProbeConcurrency: 1, + }), + ) +} + +func testRateLimit(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) { + defer wg.Done() + verifyRateLimit(ne, 1, 5000) +} + +func testThreadsAndBulkSize(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) { + defer wg.Done() + initialTemplateThreads, initialBulkSize := 1, 1 + verifyThreadsAndBulkSize(ne, initialTemplateThreads, initialBulkSize, 25, 25) +} + +func testPayloadConcurrency(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) { + defer wg.Done() + verifyPayloadConcurrency(ne, 1, 500) +} + +func verifyRateLimit(ne *nuclei.NucleiEngine, initialRate, finalRate int) { + if ne.GetExecuterOptions().RateLimiter.GetLimit() != uint(initialRate) { + panic("wrong initial rate limit") + } + time.Sleep(5 * time.Second) + ne.Options().RateLimit = finalRate + time.Sleep(20 * time.Second) + if ne.GetExecuterOptions().RateLimiter.GetLimit() != uint(finalRate) { + panic("wrong final rate limit") + } +} + +func verifyThreadsAndBulkSize(ne *nuclei.NucleiEngine, initialThreads, initialBulk, finalThreads, finalBulk int) { + if ne.Options().TemplateThreads != initialThreads || ne.Options().BulkSize != initialBulk { + panic("wrong initial standard concurrency") + } + time.Sleep(5 * time.Second) + ne.Options().TemplateThreads = finalThreads + ne.Options().BulkSize = finalBulk + time.Sleep(20 * time.Second) + if ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size != finalBulk || ne.Engine().WorkPool().Default.Size != finalThreads { + log.Fatal("wrong final concurrency", ne.Engine().WorkPool().Default.Size, finalThreads, ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size, finalBulk) + } +} + +func verifyPayloadConcurrency(ne *nuclei.NucleiEngine, initialPayloadConcurrency, finalPayloadConcurrency int) { + if ne.Options().PayloadConcurrency != initialPayloadConcurrency { + panic("wrong initial payload concurrency") + } + time.Sleep(5 * time.Second) + ne.Options().PayloadConcurrency = finalPayloadConcurrency + time.Sleep(20 * time.Second) + if ne.GetExecuterOptions().GetThreadsForNPayloadRequests(100, 0) != finalPayloadConcurrency { + panic("wrong final payload concurrency") + } +} diff --git a/go.mod b/go.mod index d2bec781..2f239bc1 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 github.com/xanzy/go-gitlab v0.84.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.21.0 + golang.org/x/net v0.24.0 golang.org/x/oauth2 v0.11.0 golang.org/x/text v0.14.0 gopkg.in/yaml.v2 v2.4.0 @@ -94,13 +94,13 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/useragent v0.0.40 - github.com/projectdiscovery/utils v0.0.88 + github.com/projectdiscovery/utils v0.0.88-0.20240404181359-663cfe2196d0 github.com/projectdiscovery/wappalyzergo v0.0.116 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 github.com/stretchr/testify v1.9.0 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 - golang.org/x/term v0.17.0 + golang.org/x/term v0.19.0 gopkg.in/yaml.v3 v3.0.1 moul.io/http2curl v1.0.0 ) @@ -142,6 +142,8 @@ require ( github.com/docker/cli v24.0.5+incompatible // indirect github.com/docker/docker v24.0.9+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect + github.com/eapache/channels v1.1.0 // indirect + github.com/eapache/queue v1.1.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect @@ -300,10 +302,10 @@ require ( go.etcd.io/bbolt v1.3.8 // indirect go.uber.org/zap v1.25.0 // indirect goftp.io/server/v2 v2.0.1 // indirect - golang.org/x/crypto v0.19.0 // indirect + golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20240119083558-1b970713d09a golang.org/x/mod v0.14.0 // indirect - golang.org/x/sys v0.17.0 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.17.0 google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 1570efb9..bf4e00c3 100644 --- a/go.sum +++ b/go.sum @@ -296,8 +296,11 @@ github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj6 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k= +github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= @@ -883,8 +886,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo= github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4= -github.com/projectdiscovery/utils v0.0.88 h1:oYfCXM+8VHNLyH/H6cOibkuDUwHUAOBAMRNPFX6NPrs= -github.com/projectdiscovery/utils v0.0.88/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA= +github.com/projectdiscovery/utils v0.0.88-0.20240404181359-663cfe2196d0 h1:2ZR0yiN0cUm/qYEMq79MfcbgM374lJSdftheYhMFxNo= +github.com/projectdiscovery/utils v0.0.88-0.20240404181359-663cfe2196d0/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA= github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8= github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -1183,8 +1186,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1277,8 +1280,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1379,8 +1382,9 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1392,8 +1396,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/runner/inputs.go b/internal/runner/inputs.go index 8dc27a7a..e7a7c29f 100644 --- a/internal/runner/inputs.go +++ b/internal/runner/inputs.go @@ -12,15 +12,14 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/utils" stringsutil "github.com/projectdiscovery/utils/strings" - "github.com/remeh/sizedwaitgroup" + syncutil "github.com/projectdiscovery/utils/sync" ) -const probeBulkSize = 50 +var GlobalProbeBulkSize = 50 // initializeTemplatesHTTPInput initializes the http form of input // for any loaded http templates if input is in non-standard format. func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { - hm, err := hybrid.New(hybrid.DefaultDiskOptions) if err != nil { return nil, errors.Wrap(err, "could not create temporary input file") @@ -31,8 +30,8 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { } gologger.Info().Msgf("Running httpx on input host") - var bulkSize = probeBulkSize - if r.options.BulkSize > probeBulkSize { + var bulkSize = GlobalProbeBulkSize + if r.options.BulkSize > GlobalProbeBulkSize { bulkSize = r.options.BulkSize } @@ -44,20 +43,29 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { return nil, errors.Wrap(err, "could not create httpx client") } + shouldFollowGlobalProbeBulkSize := bulkSize == GlobalProbeBulkSize + // Probe the non-standard URLs and store them in cache - swg := sizedwaitgroup.New(bulkSize) - count := int32(0) + swg, err := syncutil.New(syncutil.WithSize(bulkSize)) + if err != nil { + return nil, errors.Wrap(err, "could not create adaptive group") + } + var count atomic.Int32 r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool { if stringsutil.HasPrefixAny(value.Input, "http://", "https://") { return true } + if shouldFollowGlobalProbeBulkSize && swg.Size != GlobalProbeBulkSize { + swg.Resize(GlobalProbeBulkSize) + } + swg.Add() go func(input *contextargs.MetaInput) { defer swg.Done() if result := utils.ProbeURL(input.Input, httpxClient); result != "" { - atomic.AddInt32(&count, 1) + count.Add(1) _ = hm.Set(input.Input, []byte(result)) } }(value) @@ -65,6 +73,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { }) swg.Wait() - gologger.Info().Msgf("Found %d URL from httpx", atomic.LoadInt32(&count)) + gologger.Info().Msgf("Found %d URL from httpx", count.Load()) return hm, nil } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 3c6aaf0b..fa9b85bf 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -314,11 +314,17 @@ func New(options *types.Options) (*Runner, error) { } if options.RateLimitMinute > 0 { - runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimitMinute), time.Minute) - } else if options.RateLimit > 0 { - runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimit), time.Second) - } else { + gologger.Print().Msgf("[%v] %v", aurora.BrightYellow("WRN"), "rate limit per minute is deprecated - use rate-limit-duration") + options.RateLimit = options.RateLimitMinute + options.RateLimitDuration = time.Minute + } + if options.RateLimit > 0 && options.RateLimitDuration == 0 { + options.RateLimitDuration = time.Second + } + if options.RateLimit == 0 && options.RateLimitDuration == 0 { runner.rateLimiter = ratelimit.NewUnlimited(context.Background()) + } else { + runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimit), options.RateLimitDuration) } if tmpDir, err := os.MkdirTemp("", "nuclei-tmp-*"); err == nil { diff --git a/lib/config.go b/lib/config.go index 52a8e232..5b112f8d 100644 --- a/lib/config.go +++ b/lib/config.go @@ -2,6 +2,7 @@ package nuclei import ( "context" + "errors" "time" "github.com/projectdiscovery/goflags" @@ -115,17 +116,48 @@ type Concurrency struct { HeadlessTemplateConcurrency int // number of templates to run concurrently for headless templates (per host in host-spray mode) JavascriptTemplateConcurrency int // number of templates to run concurrently for javascript templates (per host in host-spray mode) TemplatePayloadConcurrency int // max concurrent payloads to run for a template (a good default is 25) + ProbeConcurrency int // max concurrent http probes to run (a good default is 50) } // WithConcurrency sets concurrency options func WithConcurrency(opts Concurrency) NucleiSDKOptions { return func(e *NucleiEngine) error { - e.opts.TemplateThreads = opts.TemplateConcurrency - e.opts.BulkSize = opts.HostConcurrency - e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency - e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency - e.opts.JsConcurrency = opts.JavascriptTemplateConcurrency - e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency + // minimum required is 1 + if opts.TemplateConcurrency <= 0 { + return errors.New("template threads must be at least 1") + } else { + e.opts.TemplateThreads = opts.TemplateConcurrency + } + if opts.HostConcurrency <= 0 { + return errors.New("host concurrency must be at least 1") + } else { + e.opts.BulkSize = opts.HostConcurrency + } + if opts.HeadlessHostConcurrency <= 0 { + return errors.New("headless host concurrency must be at least 1") + } else { + e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency + } + if opts.HeadlessTemplateConcurrency <= 0 { + return errors.New("headless template threads must be at least 1") + } else { + e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency + } + if opts.JavascriptTemplateConcurrency <= 0 { + return errors.New("js must be at least 1") + } else { + e.opts.JsConcurrency = opts.JavascriptTemplateConcurrency + } + if opts.TemplatePayloadConcurrency <= 0 { + return errors.New("payload concurrency must be at least 1") + } else { + e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency + } + if opts.ProbeConcurrency <= 0 { + return errors.New("probe concurrency must be at least 1") + } else { + e.opts.ProbeConcurrency = opts.ProbeConcurrency + } return nil } } @@ -133,7 +165,9 @@ func WithConcurrency(opts Concurrency) NucleiSDKOptions { // WithGlobalRateLimit sets global rate (i.e all hosts combined) limit options func WithGlobalRateLimit(maxTokens int, duration time.Duration) NucleiSDKOptions { return func(e *NucleiEngine) error { - e.rateLimiter = ratelimit.New(context.Background(), uint(maxTokens), duration) + e.opts.RateLimit = maxTokens + e.opts.RateLimitDuration = duration + e.rateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), e.opts.RateLimitDuration) return nil } } diff --git a/lib/multi.go b/lib/multi.go index 3a573b16..6fa791a2 100644 --- a/lib/multi.go +++ b/lib/multi.go @@ -42,11 +42,16 @@ func createEphemeralObjects(base *NucleiEngine, opts *types.Options) (*unsafeOpt Parser: base.parser, } if opts.RateLimitMinute > 0 { - u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimitMinute), time.Minute) - } else if opts.RateLimit > 0 { - u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimit), time.Second) - } else { + opts.RateLimit = opts.RateLimitMinute + opts.RateLimitDuration = time.Minute + } + if opts.RateLimit > 0 && opts.RateLimitDuration == 0 { + opts.RateLimitDuration = time.Second + } + if opts.RateLimit == 0 && opts.RateLimitDuration == 0 { u.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background()) + } else { + u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimit), opts.RateLimitDuration) } u.engine = core.New(opts) u.engine.SetExecuterOptions(u.executerOpts) diff --git a/lib/sdk.go b/lib/sdk.go index da0abae4..4bee921c 100644 --- a/lib/sdk.go +++ b/lib/sdk.go @@ -215,6 +215,14 @@ func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.Result return nil } +func (e *NucleiEngine) Options() *types.Options { + return e.opts +} + +func (e *NucleiEngine) Engine() *core.Engine { + return e.engine +} + // NewNucleiEngine creates a new nuclei engine instance func NewNucleiEngine(options ...NucleiSDKOptions) (*NucleiEngine, error) { // default options diff --git a/lib/sdk_private.go b/lib/sdk_private.go index bc66d53f..c76970c9 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -192,11 +192,16 @@ func (e *NucleiEngine) init() error { if e.executerOpts.RateLimiter == nil { if e.opts.RateLimitMinute > 0 { - e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimitMinute), time.Minute) - } else if e.opts.RateLimit > 0 { - e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), time.Second) - } else { + e.opts.RateLimit = e.opts.RateLimitMinute + e.opts.RateLimitDuration = time.Minute + } + if e.opts.RateLimit > 0 && e.opts.RateLimitDuration == 0 { + e.opts.RateLimitDuration = time.Second + } + if e.opts.RateLimit == 0 && e.opts.RateLimitDuration == 0 { e.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background()) + } else { + e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), e.opts.RateLimitDuration) } } diff --git a/pkg/core/engine.go b/pkg/core/engine.go index 93915bc2..1b4155bb 100644 --- a/pkg/core/engine.go +++ b/pkg/core/engine.go @@ -30,14 +30,19 @@ func New(options *types.Options) *Engine { return engine } -// GetWorkPool returns a workpool from options -func (e *Engine) GetWorkPool() *WorkPool { - return NewWorkPool(WorkPoolConfig{ +func (e *Engine) GetWorkPoolConfig() WorkPoolConfig { + config := WorkPoolConfig{ InputConcurrency: e.options.BulkSize, TypeConcurrency: e.options.TemplateThreads, HeadlessInputConcurrency: e.options.HeadlessBulkSize, HeadlessTypeConcurrency: e.options.HeadlessTemplateThreads, - }) + } + return config +} + +// GetWorkPool returns a workpool from options +func (e *Engine) GetWorkPool() *WorkPool { + return NewWorkPool(e.GetWorkPoolConfig()) } // SetExecuterOptions sets the executer options for the engine. This is required @@ -53,5 +58,7 @@ func (e *Engine) ExecuterOptions() protocols.ExecutorOptions { // WorkPool returns the worker pool for the engine func (e *Engine) WorkPool() *WorkPool { + // resize check point - nop if there are no changes + e.workPool.RefreshWithConfig(e.GetWorkPoolConfig()) return e.workPool } diff --git a/pkg/core/execute_options.go b/pkg/core/execute_options.go index fd1fadae..93f197fc 100644 --- a/pkg/core/execute_options.go +++ b/pkg/core/execute_options.go @@ -4,8 +4,6 @@ import ( "sync" "sync/atomic" - "github.com/remeh/sizedwaitgroup" - "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/input/provider" "github.com/projectdiscovery/nuclei/v3/pkg/output" @@ -14,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy" stringsutil "github.com/projectdiscovery/utils/strings" + syncutil "github.com/projectdiscovery/utils/sync" ) // Execute takes a list of templates/workflows that have been compiled @@ -109,9 +108,11 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe wp := e.GetWorkPool() for _, template := range templatesList { - templateType := template.Type() + // resize check point - nop if there are no changes + wp.RefreshWithConfig(e.GetWorkPoolConfig()) - var wg *sizedwaitgroup.SizedWaitGroup + templateType := template.Type() + var wg *syncutil.AdaptiveWaitGroup if templateType == types.HeadlessProtocol { wg = wp.Headless } else { @@ -134,7 +135,7 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe // executeHostSpray executes scan using host spray strategy where templates are iterated over each target func (e *Engine) executeHostSpray(templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool { results := &atomic.Bool{} - wp := sizedwaitgroup.New(e.options.BulkSize + e.options.HeadlessBulkSize) + wp, _ := syncutil.New(syncutil.WithSize(e.options.BulkSize + e.options.HeadlessBulkSize)) target.Iterate(func(value *contextargs.MetaInput) bool { wp.Add() diff --git a/pkg/core/executors.go b/pkg/core/executors.go index b491bd8e..14fb75c6 100644 --- a/pkg/core/executors.go +++ b/pkg/core/executors.go @@ -11,7 +11,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/templates" "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" generalTypes "github.com/projectdiscovery/nuclei/v3/pkg/types" - "github.com/remeh/sizedwaitgroup" + syncutil "github.com/projectdiscovery/utils/sync" ) // Executors are low level executors that deals with template execution on a target @@ -104,9 +104,9 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target return true } - wg.WaitGroup.Add() + wg.Add() go func(index uint32, skip bool, value *contextargs.MetaInput) { - defer wg.WaitGroup.Done() + defer wg.Done() defer cleanupInFlight(index) if skip { return @@ -140,7 +140,7 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target index++ return true }) - wg.WaitGroup.Wait() + wg.Wait() // on completion marks the template as completed currentInfo.Lock() @@ -158,14 +158,17 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta wp := e.GetWorkPool() for _, tpl := range alltemplates { - var sg *sizedwaitgroup.SizedWaitGroup + // resize check point - nop if there are no changes + wp.RefreshWithConfig(e.GetWorkPoolConfig()) + + var sg *syncutil.AdaptiveWaitGroup if tpl.Type() == types.HeadlessProtocol { sg = wp.Headless } else { sg = wp.Default } sg.Add() - go func(template *templates.Template, value *contextargs.MetaInput, wg *sizedwaitgroup.SizedWaitGroup) { + go func(template *templates.Template, value *contextargs.MetaInput, wg *syncutil.AdaptiveWaitGroup) { defer wg.Done() var match bool @@ -213,7 +216,10 @@ func (e *ChildExecuter) Close() *atomic.Bool { func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) { templateType := template.Type() - var wg *sizedwaitgroup.SizedWaitGroup + // resize check point - nop if there are no changes + e.e.WorkPool().RefreshWithConfig(e.e.GetWorkPoolConfig()) + + var wg *syncutil.AdaptiveWaitGroup if templateType == types.HeadlessProtocol { wg = e.e.workPool.Headless } else { diff --git a/pkg/core/workflow_execute.go b/pkg/core/workflow_execute.go index cb877cc6..19d6f0d6 100644 --- a/pkg/core/workflow_execute.go +++ b/pkg/core/workflow_execute.go @@ -5,13 +5,12 @@ import ( "net/http/cookiejar" "sync/atomic" - "github.com/remeh/sizedwaitgroup" - "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/workflows" + syncutil "github.com/projectdiscovery/utils/sync" ) const workflowStepExecutionError = "[%s] Could not execute workflow step: %s\n" @@ -32,7 +31,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b if templateThreads == 1 { templateThreads++ } - swg := sizedwaitgroup.New(templateThreads) + swg, _ := syncutil.New(syncutil.WithSize(templateThreads)) for _, template := range w.Workflows { swg.Add() @@ -40,7 +39,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b func(template *workflows.WorkflowTemplate) { defer swg.Done() - if err := e.runWorkflowStep(template, ctx, results, &swg, w); err != nil { + if err := e.runWorkflowStep(template, ctx, results, swg, w); err != nil { gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err) } }(template) @@ -51,7 +50,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b // runWorkflowStep runs a workflow step for the workflow. It executes the workflow // in a recursive manner running all subtemplates and matchers. -func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error { +func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *syncutil.AdaptiveWaitGroup, w *workflows.Workflow) error { var firstMatched bool var err error var mainErr error diff --git a/pkg/core/workpool.go b/pkg/core/workpool.go index 07117595..810f9939 100644 --- a/pkg/core/workpool.go +++ b/pkg/core/workpool.go @@ -1,9 +1,8 @@ package core import ( - "github.com/remeh/sizedwaitgroup" - "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" + syncutil "github.com/projectdiscovery/utils/sync" ) // WorkPool implements an execution pool for executing different @@ -12,8 +11,8 @@ import ( // It also allows Configuration of such requirements. This is used // for per-module like separate headless concurrency etc. type WorkPool struct { - Headless *sizedwaitgroup.SizedWaitGroup - Default *sizedwaitgroup.SizedWaitGroup + Headless *syncutil.AdaptiveWaitGroup + Default *syncutil.AdaptiveWaitGroup config WorkPoolConfig } @@ -31,13 +30,13 @@ type WorkPoolConfig struct { // NewWorkPool returns a new WorkPool instance func NewWorkPool(config WorkPoolConfig) *WorkPool { - headlessWg := sizedwaitgroup.New(config.HeadlessTypeConcurrency) - defaultWg := sizedwaitgroup.New(config.TypeConcurrency) + headlessWg, _ := syncutil.New(syncutil.WithSize(config.HeadlessTypeConcurrency)) + defaultWg, _ := syncutil.New(syncutil.WithSize(config.TypeConcurrency)) return &WorkPool{ config: config, - Headless: &headlessWg, - Default: &defaultWg, + Headless: headlessWg, + Default: defaultWg, } } @@ -47,19 +46,39 @@ func (w *WorkPool) Wait() { w.Headless.Wait() } -// InputWorkPool is a work pool per-input -type InputWorkPool struct { - WaitGroup *sizedwaitgroup.SizedWaitGroup -} - // InputPool returns a work pool for an input type -func (w *WorkPool) InputPool(templateType types.ProtocolType) *InputWorkPool { +func (w *WorkPool) InputPool(templateType types.ProtocolType) *syncutil.AdaptiveWaitGroup { var count int if templateType == types.HeadlessProtocol { count = w.config.HeadlessInputConcurrency } else { count = w.config.InputConcurrency } - swg := sizedwaitgroup.New(count) - return &InputWorkPool{WaitGroup: &swg} + swg, _ := syncutil.New(syncutil.WithSize(count)) + return swg +} + +func (w *WorkPool) RefreshWithConfig(config WorkPoolConfig) { + if w.config.TypeConcurrency != config.TypeConcurrency { + w.config.TypeConcurrency = config.TypeConcurrency + } + if w.config.HeadlessTypeConcurrency != config.HeadlessTypeConcurrency { + w.config.HeadlessTypeConcurrency = config.HeadlessTypeConcurrency + } + if w.config.InputConcurrency != config.InputConcurrency { + w.config.InputConcurrency = config.InputConcurrency + } + if w.config.HeadlessInputConcurrency != config.HeadlessInputConcurrency { + w.config.HeadlessInputConcurrency = config.HeadlessInputConcurrency + } + w.Refresh() +} + +func (w *WorkPool) Refresh() { + if w.Default.Size != w.config.TypeConcurrency { + w.Default.Resize(w.config.TypeConcurrency) + } + if w.Headless.Size != w.config.HeadlessTypeConcurrency { + w.Headless.Resize(w.config.HeadlessTypeConcurrency) + } } diff --git a/pkg/js/compiler/non-pool.go b/pkg/js/compiler/non-pool.go index 8057c496..218b89b8 100644 --- a/pkg/js/compiler/non-pool.go +++ b/pkg/js/compiler/non-pool.go @@ -4,13 +4,13 @@ import ( "sync" "github.com/dop251/goja" - "github.com/remeh/sizedwaitgroup" + syncutil "github.com/projectdiscovery/utils/sync" ) var ( - ephemeraljsc = sizedwaitgroup.New(NonPoolingVMConcurrency) + ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency)) lazyFixedSgInit = sync.OnceFunc(func() { - ephemeraljsc = sizedwaitgroup.New(NonPoolingVMConcurrency) + ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency)) }) ) diff --git a/pkg/js/compiler/pool.go b/pkg/js/compiler/pool.go index 6dba600f..678b9afb 100644 --- a/pkg/js/compiler/pool.go +++ b/pkg/js/compiler/pool.go @@ -36,7 +36,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/goconsole" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" stringsutil "github.com/projectdiscovery/utils/strings" - "github.com/remeh/sizedwaitgroup" + syncutil "github.com/projectdiscovery/utils/sync" ) const ( @@ -51,10 +51,16 @@ var ( // autoregister console node module with default printer it uses gologger backend require.RegisterNativeModule(console.ModuleName, console.RequireWithPrinter(goconsole.NewGoConsolePrinter())) }) - pooljsc sizedwaitgroup.SizedWaitGroup + pooljsc *syncutil.AdaptiveWaitGroup lazySgInit = sync.OnceFunc(func() { - pooljsc = sizedwaitgroup.New(PoolingJsVmConcurrency) + pooljsc, _ = syncutil.New(syncutil.WithSize(PoolingJsVmConcurrency)) }) + sgResizeCheck = func() { + // resize check point + if pooljsc.Size != PoolingJsVmConcurrency { + pooljsc.Resize(PoolingJsVmConcurrency) + } + } ) var gojapool = &sync.Pool{ @@ -116,6 +122,8 @@ func executeWithPoolingProgram(p *goja.Program, args *ExecuteArgs, opts *Execute // its unknown (most likely cannot be done) to limit max js runtimes at a moment without making it static // unlike sync.Pool which reacts to GC and its purposes is to reuse objects rather than creating new ones lazySgInit() + sgResizeCheck() + pooljsc.Add() defer pooljsc.Done() runtime := gojapool.Get().(*goja.Runtime) diff --git a/pkg/protocols/common/automaticscan/automaticscan.go b/pkg/protocols/common/automaticscan/automaticscan.go index 81939471..7119377b 100644 --- a/pkg/protocols/common/automaticscan/automaticscan.go +++ b/pkg/protocols/common/automaticscan/automaticscan.go @@ -30,8 +30,8 @@ import ( mapsutil "github.com/projectdiscovery/utils/maps" sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" + syncutil "github.com/projectdiscovery/utils/sync" wappalyzer "github.com/projectdiscovery/wappalyzergo" - "github.com/remeh/sizedwaitgroup" "gopkg.in/yaml.v2" ) @@ -128,7 +128,10 @@ func (s *Service) Close() bool { func (s *Service) Execute() error { gologger.Info().Msgf("Executing Automatic scan on %d target[s]", s.target.Count()) // setup host concurrency - sg := sizedwaitgroup.New(s.opts.Options.BulkSize) + sg, err := syncutil.New(syncutil.WithSize(s.opts.Options.BulkSize)) + if err != nil { + return err + } s.target.Iterate(func(value *contextargs.MetaInput) bool { sg.Add() go func(input *contextargs.MetaInput) { @@ -246,7 +249,7 @@ func (s *Service) getTagsUsingDetectionTemplates(input *contextargs.MetaInput) ( // execute tech detection templates on target tags := map[string]struct{}{} m := &sync.Mutex{} - sg := sizedwaitgroup.New(s.opts.Options.TemplateThreads) + sg, _ := syncutil.New(syncutil.WithSize(s.opts.Options.TemplateThreads)) counter := atomic.Uint32{} for _, t := range s.techTemplates { diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go index 280e8161..d4e70e13 100644 --- a/pkg/protocols/dns/request.go +++ b/pkg/protocols/dns/request.go @@ -9,7 +9,6 @@ import ( "github.com/miekg/dns" "github.com/pkg/errors" - "github.com/remeh/sizedwaitgroup" "go.uber.org/multierr" "golang.org/x/exp/maps" @@ -27,6 +26,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/utils" "github.com/projectdiscovery/retryabledns" iputil "github.com/projectdiscovery/utils/ip" + syncutil "github.com/projectdiscovery/utils/sync" ) var _ protocols.Request = &Request{} @@ -62,9 +62,15 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, variablesMap := request.options.Variables.Evaluate(vars) vars = generators.MergeMaps(vars, variablesMap, request.options.Constants) + // if request threads matches global payload concurrency we follow it + shouldFollowGlobal := request.Threads == request.options.Options.PayloadConcurrency + if request.generator != nil { iterator := request.generator.NewIterator() - swg := sizedwaitgroup.New(request.Threads) + swg, err := syncutil.New(syncutil.WithSize(request.Threads)) + if err != nil { + return err + } var multiErr error m := &sync.Mutex{} @@ -73,6 +79,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, if !ok { break } + + // resize check point - nop if there are no changes + if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency { + swg.Resize(request.options.Options.PayloadConcurrency) + } + value = generators.MergeMaps(vars, value) swg.Add() go func(newVars map[string]interface{}) { @@ -140,7 +152,7 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad } } - request.options.RateLimiter.Take() + request.options.RateLimitTake() // Send the request to the target servers response, err := dnsClient.Do(compiledRequest) diff --git a/pkg/protocols/file/request.go b/pkg/protocols/file/request.go index eb73544f..f13f08d1 100644 --- a/pkg/protocols/file/request.go +++ b/pkg/protocols/file/request.go @@ -11,7 +11,6 @@ import ( "github.com/docker/go-units" "github.com/mholt/archiver" "github.com/pkg/errors" - "github.com/remeh/sizedwaitgroup" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/operators" @@ -24,6 +23,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" sliceutil "github.com/projectdiscovery/utils/slice" + syncutil "github.com/projectdiscovery/utils/sync" ) var _ protocols.Request = &Request{} @@ -47,8 +47,11 @@ var errEmptyResult = errors.New("Empty result") // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - wg := sizedwaitgroup.New(request.options.Options.BulkSize) - err := request.getInputPaths(input.MetaInput.Input, func(filePath string) { + wg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize)) + if err != nil { + return err + } + err = request.getInputPaths(input.MetaInput.Input, func(filePath string) { wg.Add() func(filePath string) { defer wg.Done() diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index 7338db44..348cab0a 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -214,7 +214,7 @@ func geTimeParameter(p *Page, act *Action, parameterName string, defaultValue ti } // ActionAddHeader executes a AddHeader action. -func (p *Page) ActionAddHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error { +func (p *Page) ActionAddHeader(act *Action, out map[string]string) error { in := p.getActionArgWithDefaultValues(act, "part") args := make(map[string]string) @@ -225,7 +225,7 @@ func (p *Page) ActionAddHeader(act *Action, out map[string]string /*TODO review } // ActionSetHeader executes a SetHeader action. -func (p *Page) ActionSetHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error { +func (p *Page) ActionSetHeader(act *Action, out map[string]string) error { in := p.getActionArgWithDefaultValues(act, "part") args := make(map[string]string) @@ -236,7 +236,7 @@ func (p *Page) ActionSetHeader(act *Action, out map[string]string /*TODO review } // ActionDeleteHeader executes a DeleteHeader action. -func (p *Page) ActionDeleteHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error { +func (p *Page) ActionDeleteHeader(act *Action, out map[string]string) error { in := p.getActionArgWithDefaultValues(act, "part") args := make(map[string]string) @@ -343,7 +343,7 @@ func (p *Page) RunScript(action *Action, out map[string]string) error { } // ClickElement executes click actions for an element. -func (p *Page) ClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { +func (p *Page) ClickElement(act *Action, out map[string]string) error { element, err := p.pageElementBy(act.Data) if err != nil { return errors.Wrap(err, errCouldNotGetElement) @@ -358,12 +358,12 @@ func (p *Page) ClickElement(act *Action, out map[string]string /*TODO review unu } // KeyboardAction executes a keyboard action on the page. -func (p *Page) KeyboardAction(act *Action, out map[string]string /*TODO review unused parameter*/) error { +func (p *Page) KeyboardAction(act *Action, out map[string]string) error { return p.page.Keyboard.Type([]input.Key(p.getActionArgWithDefaultValues(act, "keys"))...) } // RightClickElement executes right click actions for an element. -func (p *Page) RightClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { +func (p *Page) RightClickElement(act *Action, out map[string]string) error { element, err := p.pageElementBy(act.Data) if err != nil { return errors.Wrap(err, errCouldNotGetElement) @@ -441,7 +441,7 @@ func (p *Page) Screenshot(act *Action, out map[string]string) error { } // InputElement executes input element actions for an element. -func (p *Page) InputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { +func (p *Page) InputElement(act *Action, out map[string]string) error { value := p.getActionArgWithDefaultValues(act, "value") if value == "" { return errinvalidArguments @@ -460,7 +460,7 @@ func (p *Page) InputElement(act *Action, out map[string]string /*TODO review unu } // TimeInputElement executes time input on an element -func (p *Page) TimeInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { +func (p *Page) TimeInputElement(act *Action, out map[string]string) error { value := p.getActionArgWithDefaultValues(act, "value") if value == "" { return errinvalidArguments @@ -483,7 +483,7 @@ func (p *Page) TimeInputElement(act *Action, out map[string]string /*TODO review } // SelectInputElement executes select input statement action on a element -func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { +func (p *Page) SelectInputElement(act *Action, out map[string]string) error { value := p.getActionArgWithDefaultValues(act, "value") if value == "" { return errinvalidArguments @@ -508,7 +508,7 @@ func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO revi } // WaitLoad waits for the page to load -func (p *Page) WaitLoad(act *Action, out map[string]string /*TODO review unused parameter*/) error { +func (p *Page) WaitLoad(act *Action, out map[string]string) error { p.page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)() // Wait for the window.onload event and also wait for the network requests @@ -538,7 +538,7 @@ func (p *Page) GetResource(act *Action, out map[string]string) error { } // FilesInput acts with a file input element on page -func (p *Page) FilesInput(act *Action, out map[string]string /*TODO review unused parameter*/) error { +func (p *Page) FilesInput(act *Action, out map[string]string) error { element, err := p.pageElementBy(act.Data) if err != nil { return errors.Wrap(err, errCouldNotGetElement) @@ -589,7 +589,7 @@ func (p *Page) ExtractElement(act *Action, out map[string]string) error { } // WaitEvent waits for an event to happen on the page. -func (p *Page) WaitEvent(act *Action, out map[string]string /*TODO review unused parameter*/) (func() error, error) { +func (p *Page) WaitEvent(act *Action, out map[string]string) (func() error, error) { event := p.getActionArgWithDefaultValues(act, "event") if event == "" { return nil, errors.New("event not recognized") @@ -661,14 +661,14 @@ func (p *Page) pageElementBy(data map[string]string) (*rod.Element, error) { } // DebugAction enables debug action on a page. -func (p *Page) DebugAction(act *Action, out map[string]string /*TODO review unused parameter*/) error { +func (p *Page) DebugAction(act *Action, out map[string]string) error { p.instance.browser.engine.SlowMotion(5 * time.Second) p.instance.browser.engine.Trace(true) return nil } // SleepAction sleeps on the page for a specified duration -func (p *Page) SleepAction(act *Action, out map[string]string /*TODO review unused parameter*/) error { +func (p *Page) SleepAction(act *Action, out map[string]string) error { seconds := act.Data["duration"] if seconds == "" { seconds = "5" diff --git a/pkg/protocols/http/httputils/spm.go b/pkg/protocols/http/httputils/spm.go index ccaa9a85..52d13f06 100644 --- a/pkg/protocols/http/httputils/spm.go +++ b/pkg/protocols/http/httputils/spm.go @@ -4,7 +4,7 @@ import ( "context" "sync" - "github.com/remeh/sizedwaitgroup" + syncutil "github.com/projectdiscovery/utils/sync" ) // WorkPoolType is the type of work pool to use @@ -26,7 +26,7 @@ type StopAtFirstMatchHandler[T any] struct { // work pool and its type poolType WorkPoolType - sgPool sizedwaitgroup.SizedWaitGroup + sgPool *syncutil.AdaptiveWaitGroup wgPool *sync.WaitGroup // internal / unexported @@ -40,10 +40,13 @@ type StopAtFirstMatchHandler[T any] struct { // NewBlockingSPMHandler creates a new stop at first match handler func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *StopAtFirstMatchHandler[T] { ctx1, cancel := context.WithCancel(ctx) + + awg, _ := syncutil.New(syncutil.WithSize(size)) + s := &StopAtFirstMatchHandler[T]{ ResultChan: make(chan T, 1), poolType: Blocking, - sgPool: sizedwaitgroup.New(size), + sgPool: awg, internalWg: &sync.WaitGroup{}, ctx: ctx1, cancel: cancel, @@ -140,6 +143,16 @@ func (h *StopAtFirstMatchHandler[T]) Release() { } } +func (h *StopAtFirstMatchHandler[T]) Resize(size int) { + if h.sgPool.Size != size { + h.sgPool.Resize(size) + } +} + +func (h *StopAtFirstMatchHandler[T]) Size() int { + return h.sgPool.Size +} + // Wait waits for all work to be done func (h *StopAtFirstMatchHandler[T]) Wait() { switch h.poolType { diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 2763c5c6..5d64216f 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -165,6 +165,9 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV // Workers that keeps enqueuing new requests maxWorkers := request.Threads + // if request threads matches global payload concurrency we follow it + shouldFollowGlobal := maxWorkers == request.options.Options.PayloadConcurrency + if protocolstate.IsLowOnMemory() { maxWorkers = protocolstate.GuardThreadsOrDefault(request.Threads) } @@ -198,6 +201,12 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV if !ok { break } + + // resize check point - nop if there are no changes + if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency { + spmHandler.Resize(request.options.Options.PayloadConcurrency) + } + ctx := request.newContext(input) generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues) if err != nil { @@ -222,7 +231,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV return case spmHandler.ResultChan <- func() error { // putting ratelimiter here prevents any unnecessary waiting if any - request.options.RateLimiter.Take() + request.options.RateLimitTake() previous := make(map[string]interface{}) return request.executeRequest(input, httpRequest, previous, false, wrappedCallback, 0) }(): @@ -366,7 +375,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa executeFunc := func(data string, payloads, dynamicValue map[string]interface{}) (bool, error) { hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators) - request.options.RateLimiter.Take() + request.options.RateLimitTake() ctx := request.newContext(input) ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(request.options.Options.Timeout)*time.Second) diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go index d6a29c18..aed17f54 100644 --- a/pkg/protocols/http/request_fuzz.go +++ b/pkg/protocols/http/request_fuzz.go @@ -145,7 +145,7 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) { return false } - request.options.RateLimiter.Take() + request.options.RateLimitTake() req := &generatedRequest{ request: gr.Request, dynamicValues: gr.DynamicValues, diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index dee3ff35..54bb3788 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -34,8 +34,8 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/types" errorutil "github.com/projectdiscovery/utils/errors" iputil "github.com/projectdiscovery/utils/ip" + syncutil "github.com/projectdiscovery/utils/sync" urlutil "github.com/projectdiscovery/utils/url" - "github.com/remeh/sizedwaitgroup" ) // Request is a request for the javascript protocol @@ -406,7 +406,11 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo requestOptions := request.options gotmatches := &atomic.Bool{} - sg := sizedwaitgroup.New(threads) + // if request threads matches global payload concurrency we follow it + shouldFollowGlobal := threads == request.options.Options.PayloadConcurrency + + sg, _ := syncutil.New(syncutil.WithSize(threads)) + if request.generator != nil { iterator := request.generator.NewIterator() for { @@ -414,6 +418,12 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo if !ok { break } + + // resize check point - nop if there are no changes + if shouldFollowGlobal && sg.Size != request.options.Options.PayloadConcurrency { + sg.Resize(request.options.Options.PayloadConcurrency) + } + sg.Add() go func() { defer sg.Done() diff --git a/pkg/protocols/network/networkclientpool/clientpool.go b/pkg/protocols/network/networkclientpool/clientpool.go index 1a933413..a67cee29 100644 --- a/pkg/protocols/network/networkclientpool/clientpool.go +++ b/pkg/protocols/network/networkclientpool/clientpool.go @@ -11,7 +11,7 @@ var ( ) // Init initializes the clientpool implementation -func Init(options *types.Options /*TODO review unused parameter*/) error { +func Init(options *types.Options) error { // Don't create clients if already created in the past. if normalClient != nil { return nil diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go index 01991c04..146c6570 100644 --- a/pkg/protocols/network/request.go +++ b/pkg/protocols/network/request.go @@ -12,7 +12,6 @@ import ( "time" "github.com/pkg/errors" - "github.com/remeh/sizedwaitgroup" "go.uber.org/multierr" "golang.org/x/exp/maps" @@ -34,6 +33,7 @@ import ( errorutil "github.com/projectdiscovery/utils/errors" mapsutil "github.com/projectdiscovery/utils/maps" "github.com/projectdiscovery/utils/reader" + syncutil "github.com/projectdiscovery/utils/sync" ) var ( @@ -174,17 +174,29 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA return err } + // if request threads matches global payload concurrency we follow it + shouldFollowGlobal := request.Threads == request.options.Options.PayloadConcurrency + if request.generator != nil { iterator := request.generator.NewIterator() var multiErr error m := &sync.Mutex{} - swg := sizedwaitgroup.New(request.Threads) + swg, err := syncutil.New(syncutil.WithSize(request.Threads)) + if err != nil { + return err + } for { value, ok := iterator.Value() if !ok { break } + + // resize check point - nop if there are no changes + if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency { + swg.Resize(request.options.Options.PayloadConcurrency) + } + value = generators.MergeMaps(value, payloads) swg.Add() go func(vars map[string]interface{}) { diff --git a/pkg/protocols/offlinehttp/request.go b/pkg/protocols/offlinehttp/request.go index 7c64859e..4a440c16 100644 --- a/pkg/protocols/offlinehttp/request.go +++ b/pkg/protocols/offlinehttp/request.go @@ -6,7 +6,6 @@ import ( "os" "github.com/pkg/errors" - "github.com/remeh/sizedwaitgroup" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/output" @@ -17,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/utils/conversion" + syncutil "github.com/projectdiscovery/utils/sync" ) var _ protocols.Request = &Request{} @@ -29,10 +29,13 @@ func (request *Request) Type() templateTypes.ProtocolType { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - wg := sizedwaitgroup.New(request.options.Options.BulkSize) +func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + wg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize)) + if err != nil { + return err + } - err := request.getInputPaths(input.MetaInput.Input, func(data string) { + err = request.getInputPaths(input.MetaInput.Input, func(data string) { wg.Add() go func(data string) { diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index f9d90642..6328bc36 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -34,9 +34,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/types" ) -// Optional Callback to update Thread count in payloads across all requests -type PayloadThreadSetterCallback func(opts *ExecutorOptions, totalRequests, currentThreads int) int - var ( MaxTemplateFileSizeForEncoding = 1024 * 1024 ) @@ -114,10 +111,6 @@ type ExecutorOptions struct { // JsCompiler is abstracted javascript compiler which adds node modules and provides execution // environment for javascript templates JsCompiler *compiler.Compiler - // Optional Callback function to update Thread count in payloads across all protocols - // based on given logic. by default nuclei reverts to using value of `-c` when threads count - // is not specified or is 0 in template - OverrideThreadsCount PayloadThreadSetterCallback // AuthProvider is a provider for auth strategies AuthProvider authprovider.AuthProvider //TemporaryDirectory is the directory to store temporary files @@ -128,17 +121,25 @@ type ExecutorOptions struct { ExportReqURLPattern bool } +// todo: centralizing components is not feasible with current clogged architecture +// a possible approach could be an internal event bus with pub-subs? This would be less invasive than +// reworking dep injection from scratch +func (eo *ExecutorOptions) RateLimitTake() { + if eo.RateLimiter.GetLimit() != uint(eo.Options.RateLimit) { + eo.RateLimiter.SetLimit(uint(eo.Options.RateLimit)) + eo.RateLimiter.SetDuration(eo.Options.RateLimitDuration) + } + eo.RateLimiter.Take() +} + // GetThreadsForPayloadRequests returns the number of threads to use as default for // given max-request of payloads func (e *ExecutorOptions) GetThreadsForNPayloadRequests(totalRequests int, currentThreads int) int { - if e.OverrideThreadsCount != nil { - return e.OverrideThreadsCount(e, totalRequests, currentThreads) - } if currentThreads > 0 { return currentThreads - } else { - return e.Options.PayloadConcurrency } + + return e.Options.PayloadConcurrency } // CreateTemplateCtxStore creates template context store (which contains templateCtx for every scan) diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index 96d68e1d..e59aa015 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -54,6 +54,7 @@ var DefaultOptions = &types.Options{ Timeout: 5, Retries: 1, RateLimit: 150, + RateLimitDuration: time.Second, ProjectPath: "", Severities: severity.Severities{}, Targets: []string{}, diff --git a/pkg/tmplexec/flow/vm.go b/pkg/tmplexec/flow/vm.go index 2e22bd8e..f1f7dbb8 100644 --- a/pkg/tmplexec/flow/vm.go +++ b/pkg/tmplexec/flow/vm.go @@ -1,6 +1,7 @@ package flow import ( + "context" "reflect" "sync" @@ -12,33 +13,10 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow/builtin" "github.com/projectdiscovery/nuclei/v3/pkg/types" - "github.com/remeh/sizedwaitgroup" + "github.com/projectdiscovery/utils/sync/sizedpool" ) -type jsWaitGroup struct { - sync.Once - sg sizedwaitgroup.SizedWaitGroup -} - -var jsPool = &jsWaitGroup{} - -// GetJSRuntime returns a new JS runtime from pool -func GetJSRuntime(opts *types.Options) *goja.Runtime { - jsPool.Do(func() { - if opts.JsConcurrency < 100 { - opts.JsConcurrency = 100 - } - jsPool.sg = sizedwaitgroup.New(opts.JsConcurrency) - }) - jsPool.sg.Add() - return gojapool.Get().(*goja.Runtime) -} - -// PutJSRuntime returns a JS runtime to pool -func PutJSRuntime(runtime *goja.Runtime) { - defer jsPool.sg.Done() - gojapool.Put(runtime) -} +var jsOnce sync.Once // js runtime pool using sync.Pool var gojapool = &sync.Pool{ @@ -49,8 +27,29 @@ var gojapool = &sync.Pool{ }, } -func registerBuiltins(runtime *goja.Runtime) { +var sizedgojapool *sizedpool.SizedPool[*goja.Runtime] +// GetJSRuntime returns a new JS runtime from pool +func GetJSRuntime(opts *types.Options) *goja.Runtime { + jsOnce.Do(func() { + if opts.JsConcurrency < 100 { + opts.JsConcurrency = 100 + } + sizedgojapool, _ = sizedpool.New[*goja.Runtime]( + sizedpool.WithPool[*goja.Runtime](gojapool), + sizedpool.WithSize[*goja.Runtime](int64(opts.JsConcurrency)), + ) + }) + runtime, _ := sizedgojapool.Get(context.TODO()) + return runtime +} + +// PutJSRuntime returns a JS runtime to pool +func PutJSRuntime(runtime *goja.Runtime) { + sizedgojapool.Put(runtime) +} + +func registerBuiltins(runtime *goja.Runtime) { _ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ Name: "log", Description: "Logs a given object/message to stdout (only for debugging purposes)", diff --git a/pkg/types/types.go b/pkg/types/types.go index ce783809..81a27316 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -132,7 +132,10 @@ type Options struct { Retries int // Rate-Limit is the maximum number of requests per specified target RateLimit int + // Rate Limit Duration interval between burst resets + RateLimitDuration time.Duration // Rate-Limit is the maximum number of requests per minute for specified target + // Deprecated: Use RateLimitDuration - automatically set Rate Limit Duration to 60 seconds RateLimitMinute int // PageTimeout is the maximum time to wait for a page in seconds PageTimeout int @@ -382,6 +385,8 @@ type Options struct { SkipFormatValidation bool // PayloadConcurrency is the number of concurrent payloads to run per template PayloadConcurrency int + // ProbeConcurrency is the number of concurrent http probes to run with httpx + ProbeConcurrency int // Dast only runs DAST templates DAST bool } @@ -410,6 +415,7 @@ func (options *Options) HasClientCertificates() bool { func DefaultOptions() *Options { return &Options{ RateLimit: 150, + RateLimitDuration: time.Second, BulkSize: 25, TemplateThreads: 25, HeadlessBulkSize: 10,