mirror of https://github.com/daffainfo/nuclei.git
internal sync fix + speed up
Mzack9999
parent
aa06c9ef17
commit
d0a0c6d0c3
|
|
@ -1,6 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
|
|
@ -9,11 +10,32 @@ import (
|
|||
)
|
||||
|
||||
func main() {
|
||||
ne, err := nuclei.NewNucleiEngine(
|
||||
nuclei.WithTemplateFilters(nuclei.TemplateFilters{
|
||||
IDs: []string{"header-command-injection"},
|
||||
IncludeTags: []string{"fuzz"},
|
||||
}),
|
||||
ne, err := initializeNucleiEngine()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer ne.Close()
|
||||
|
||||
ne.LoadTargets([]string{"http://honey.scanme.sh"}, false)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(3)
|
||||
|
||||
go testRateLimit(&wg, ne)
|
||||
go testThreadsAndBulkSize(&wg, ne)
|
||||
go testPayloadConcurrency(&wg, ne)
|
||||
|
||||
err = ne.ExecuteWithCallback(nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func initializeNucleiEngine() (*nuclei.NucleiEngine, error) {
|
||||
return nuclei.NewNucleiEngine(
|
||||
nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "http"}),
|
||||
nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}),
|
||||
nuclei.WithGlobalRateLimit(1, time.Second),
|
||||
nuclei.WithConcurrency(nuclei.Concurrency{
|
||||
|
|
@ -25,78 +47,57 @@ func main() {
|
|||
TemplatePayloadConcurrency: 1,
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// load targets and optionally probe non http/https targets
|
||||
ne.LoadTargets([]string{"http://honey.scanme.sh"}, false)
|
||||
}
|
||||
|
||||
var wgtest sync.WaitGroup
|
||||
func testRateLimit(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
|
||||
defer wg.Done()
|
||||
verifyRateLimit(ne, 1, 5000)
|
||||
}
|
||||
|
||||
// speed tests
|
||||
// increase rate limit
|
||||
wgtest.Add(1)
|
||||
go func() {
|
||||
defer wgtest.Done()
|
||||
initialRate := ne.GetExecuterOptions().RateLimiter.GetLimit()
|
||||
if initialRate != 1 {
|
||||
func testThreadsAndBulkSize(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
|
||||
defer wg.Done()
|
||||
initialTemplateThreads, initialBulkSize := 1, 1
|
||||
verifyThreadsAndBulkSize(ne, initialTemplateThreads, initialBulkSize, 25, 25)
|
||||
}
|
||||
|
||||
func testPayloadConcurrency(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
|
||||
defer wg.Done()
|
||||
verifyPayloadConcurrency(ne, 1, 500)
|
||||
}
|
||||
|
||||
func verifyRateLimit(ne *nuclei.NucleiEngine, initialRate, finalRate int) {
|
||||
if ne.GetExecuterOptions().RateLimiter.GetLimit() != uint(initialRate) {
|
||||
panic("wrong initial rate limit")
|
||||
}
|
||||
time.Sleep(10 * time.Second)
|
||||
ne.Options().RateLimit = 1000
|
||||
time.Sleep(10 * time.Second)
|
||||
finalRate := ne.GetExecuterOptions().RateLimiter.GetLimit()
|
||||
if finalRate != 1000 {
|
||||
time.Sleep(5 * time.Second)
|
||||
ne.Options().RateLimit = finalRate
|
||||
time.Sleep(20 * time.Second)
|
||||
if ne.GetExecuterOptions().RateLimiter.GetLimit() != uint(finalRate) {
|
||||
panic("wrong final rate limit")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// increase threads and bulk size
|
||||
wgtest.Add(1)
|
||||
go func() {
|
||||
defer wgtest.Done()
|
||||
initialTemplateThreads := ne.Options().TemplateThreads
|
||||
initialBulkSize := ne.Options().BulkSize
|
||||
if initialTemplateThreads != 1 || initialBulkSize != 1 {
|
||||
func verifyThreadsAndBulkSize(ne *nuclei.NucleiEngine, initialThreads, initialBulk, finalThreads, finalBulk int) {
|
||||
if ne.Options().TemplateThreads != initialThreads || ne.Options().BulkSize != initialBulk {
|
||||
panic("wrong initial standard concurrency")
|
||||
}
|
||||
time.Sleep(10 * time.Second)
|
||||
ne.Options().TemplateThreads = 5
|
||||
ne.Options().BulkSize = 25
|
||||
time.Sleep(10 * time.Second)
|
||||
// check new values via workpool
|
||||
finalTemplateThreads := ne.Engine().WorkPool().Default.Size
|
||||
finalBulkSize := ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size
|
||||
if finalTemplateThreads != 5 && finalBulkSize != 25 {
|
||||
panic("wrong final concurreny")
|
||||
time.Sleep(5 * time.Second)
|
||||
ne.Options().TemplateThreads = finalThreads
|
||||
ne.Options().BulkSize = finalBulk
|
||||
time.Sleep(20 * time.Second)
|
||||
if ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size != finalBulk || ne.Engine().WorkPool().Default.Size != finalThreads {
|
||||
log.Fatal("wrong final concurrency", ne.Engine().WorkPool().Default.Size, finalThreads, ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size, finalBulk)
|
||||
}
|
||||
}
|
||||
|
||||
func verifyPayloadConcurrency(ne *nuclei.NucleiEngine, initialPayloadConcurrency, finalPayloadConcurrency int) {
|
||||
if ne.Options().PayloadConcurrency != initialPayloadConcurrency {
|
||||
panic("wrong initial payload concurrency")
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
ne.Options().PayloadConcurrency = finalPayloadConcurrency
|
||||
time.Sleep(20 * time.Second)
|
||||
if ne.GetExecuterOptions().GetThreadsForNPayloadRequests(100, 0) != finalPayloadConcurrency {
|
||||
panic("wrong final payload concurrency")
|
||||
}
|
||||
}()
|
||||
|
||||
// increase payload concurrency
|
||||
wgtest.Add(1)
|
||||
go func() {
|
||||
defer wgtest.Done()
|
||||
initialpayloadConcurrency := ne.Options().PayloadConcurrency
|
||||
if initialpayloadConcurrency != 1 {
|
||||
panic("wrong initial payload concurrency")
|
||||
}
|
||||
time.Sleep(10 * time.Second)
|
||||
ne.Options().PayloadConcurrency = 100
|
||||
time.Sleep(10 * time.Second)
|
||||
|
||||
// the ongoing and next payload iterations will retrieve parallelism from this function
|
||||
// it should have the new set value, that will be cascade applied to all running adaptive wait groups
|
||||
finalPayloadConcurrency := ne.GetExecuterOptions().GetThreadsForNPayloadRequests(100, 0)
|
||||
if finalPayloadConcurrency != 100 {
|
||||
panic("wrong initial payload concurrency")
|
||||
}
|
||||
}()
|
||||
|
||||
err = ne.ExecuteWithCallback(nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer ne.Close()
|
||||
|
||||
wgtest.Wait()
|
||||
}
|
||||
|
|
|
|||
8
go.mod
8
go.mod
|
|
@ -38,7 +38,7 @@ require (
|
|||
github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222
|
||||
github.com/xanzy/go-gitlab v0.84.0
|
||||
go.uber.org/multierr v1.11.0
|
||||
golang.org/x/net v0.21.0
|
||||
golang.org/x/net v0.24.0
|
||||
golang.org/x/oauth2 v0.11.0
|
||||
golang.org/x/text v0.14.0
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
|
|
@ -100,7 +100,7 @@ require (
|
|||
github.com/seh-msft/burpxml v1.0.1
|
||||
github.com/stretchr/testify v1.9.0
|
||||
github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
|
||||
golang.org/x/term v0.17.0
|
||||
golang.org/x/term v0.19.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
moul.io/http2curl v1.0.0
|
||||
)
|
||||
|
|
@ -302,10 +302,10 @@ require (
|
|||
go.etcd.io/bbolt v1.3.8 // indirect
|
||||
go.uber.org/zap v1.25.0 // indirect
|
||||
goftp.io/server/v2 v2.0.1 // indirect
|
||||
golang.org/x/crypto v0.19.0 // indirect
|
||||
golang.org/x/crypto v0.22.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
|
||||
golang.org/x/mod v0.14.0 // indirect
|
||||
golang.org/x/sys v0.17.0 // indirect
|
||||
golang.org/x/sys v0.19.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
golang.org/x/tools v0.17.0
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
|
|
|
|||
15
go.sum
15
go.sum
|
|
@ -1186,8 +1186,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
|
|||
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
|
||||
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
|
||||
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
|
||||
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
|
||||
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
|
|
@ -1280,8 +1280,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
|
|||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
|
||||
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
|
||||
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
@ -1382,8 +1382,9 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
|||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
|
||||
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
|
|
@ -1395,8 +1396,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
|||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
|
||||
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
|
||||
golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
|
||||
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
|
|||
|
|
@ -58,5 +58,7 @@ func (e *Engine) ExecuterOptions() protocols.ExecutorOptions {
|
|||
|
||||
// WorkPool returns the worker pool for the engine
|
||||
func (e *Engine) WorkPool() *WorkPool {
|
||||
// resize check point - nop if there are no changes
|
||||
e.workPool.RefreshWithConfig(e.GetWorkPoolConfig())
|
||||
return e.workPool
|
||||
}
|
||||
|
|
|
|||
|
|
@ -217,7 +217,7 @@ func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs
|
|||
templateType := template.Type()
|
||||
|
||||
// resize check point - nop if there are no changes
|
||||
e.e.workPool.RefreshWithConfig(e.e.GetWorkPoolConfig())
|
||||
e.e.WorkPool().RefreshWithConfig(e.e.GetWorkPoolConfig())
|
||||
|
||||
var wg *syncutil.AdaptiveWaitGroup
|
||||
if templateType == types.HeadlessProtocol {
|
||||
|
|
|
|||
Loading…
Reference in New Issue