diff --git a/go.mod b/go.mod index be701290..31de6169 100644 --- a/go.mod +++ b/go.mod @@ -21,16 +21,16 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 github.com/projectdiscovery/fastdialer v0.0.60 - github.com/projectdiscovery/hmap v0.0.39 + github.com/projectdiscovery/hmap v0.0.40 github.com/projectdiscovery/interactsh v1.1.8 github.com/projectdiscovery/rawhttp v0.1.35 - github.com/projectdiscovery/retryabledns v1.0.56 + github.com/projectdiscovery/retryabledns v1.0.57 github.com/projectdiscovery/retryablehttp-go v1.0.49 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/segmentio/ksuid v1.0.4 - github.com/shirou/gopsutil/v3 v3.23.7 // indirect + github.com/shirou/gopsutil/v3 v3.24.2 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.5.1 github.com/syndtr/goleveldb v1.0.0 @@ -59,6 +59,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.13.27 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 + github.com/cespare/xxhash/v2 v2.2.0 github.com/charmbracelet/glamour v0.6.0 github.com/denisenkom/go-mssqldb v0.12.3 github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c @@ -91,7 +92,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/useragent v0.0.39 - github.com/projectdiscovery/utils v0.0.80 + github.com/projectdiscovery/utils v0.0.83-0.20240305000020-ff30de2464cd github.com/projectdiscovery/wappalyzergo v0.0.111 github.com/redis/go-redis/v9 v9.1.0 github.com/sashabaranov/go-openai v1.15.3 @@ -123,7 +124,6 @@ require ( github.com/bits-and-blooms/bloom/v3 v3.5.0 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/cfssl v1.6.4 // indirect @@ -132,7 +132,6 @@ require ( github.com/corpix/uarand v0.2.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect - github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.10.0 // indirect github.com/docker/cli v24.0.5+incompatible // indirect @@ -188,9 +187,11 @@ require ( github.com/projectdiscovery/asnmap v1.0.6 // indirect github.com/projectdiscovery/cdncheck v1.0.9 // indirect github.com/projectdiscovery/freeport v0.0.5 // indirect + github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect github.com/projectdiscovery/stringsutil v0.0.2 // indirect github.com/quic-go/quic-go v0.40.1 // indirect github.com/refraction-networking/utls v1.6.1 // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.2.1 // indirect @@ -270,8 +271,8 @@ require ( github.com/projectdiscovery/networkpolicy v0.0.7 github.com/rivo/uniseg v0.4.4 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect - github.com/tklauser/go-sysconf v0.3.11 // indirect - github.com/tklauser/numcpus v0.6.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/trivago/tgo v1.0.7 github.com/ulikunitz/xz v0.5.11 // indirect github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect @@ -280,7 +281,7 @@ require ( github.com/ysmood/goob v0.4.0 // indirect github.com/ysmood/gson v0.7.3 // indirect github.com/ysmood/leakless v0.8.0 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c // indirect go.etcd.io/bbolt v1.3.8 // indirect diff --git a/go.sum b/go.sum index dcfa4ae2..7025d203 100644 --- a/go.sum +++ b/go.sum @@ -260,8 +260,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= -github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= -github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw= github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -830,12 +828,14 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.1 h1:f08ZnYlbDZV/TNGDvIXV9s/oB/sAI+HWaSbW4em4aKM= github.com/projectdiscovery/gozero v0.0.1/go.mod h1:/dHwbly+1lhOX9UreVure4lEe7K4hIHeu/c/wZGNTDo= -github.com/projectdiscovery/hmap v0.0.39 h1:R33JJzPz8TMUm1TJQ/X5zVJbmyTS64EttZ085thq19g= -github.com/projectdiscovery/hmap v0.0.39/go.mod h1:wEPoEIVGPdHsc9EnE+ERUdgNyp29zZn6gACOALylHOg= +github.com/projectdiscovery/hmap v0.0.40 h1:WGAIXXMY2vbV0ep7Q8s27Up/ejs8Wo1hh5AEhynLfmw= +github.com/projectdiscovery/hmap v0.0.40/go.mod h1:5JkQW9t/UNK95YEY6irOXHqrS/xVBUtrYEhtq61ttII= github.com/projectdiscovery/httpx v1.3.9 h1:jDdoGH+5VVU/jI6dnai1DKNw9USPyCcw+tDh4RCVQ2g= github.com/projectdiscovery/httpx v1.3.9/go.mod h1:a/a5X6e2NLnS/+b3buFadGUpZSolnVkMA7KZdpCdg58= github.com/projectdiscovery/interactsh v1.1.8 h1:mDD+f/oo2tV4Z1WyUync0tgYeJyuiS89Un64Gm6Pvgk= github.com/projectdiscovery/interactsh v1.1.8/go.mod h1:E20ywFb7bL01GcOOk+6VZF48XZ8AZvYvBpULoBUSTbg= +github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= +github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.16 h1:rjj1w5D6hbTsUQXYClLcGdfBEy9bryclgi70t0vBggo= github.com/projectdiscovery/mapcidr v1.1.16/go.mod h1:rGqpBhStdwOQ2uS62QM9qPsybwMwIhT7CTd2bxoHs8Q= github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8= @@ -848,8 +848,8 @@ github.com/projectdiscovery/rawhttp v0.1.35 h1:9Hkbu1WLN5coj6+HBaqi26PjMNFnw1XrM github.com/projectdiscovery/rawhttp v0.1.35/go.mod h1:9mS0N3BfOBYwQWgyI+bXBaFVMFBtJVTcZF0FENea7mA= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.56 h1:Rk/fvBSNjw4vzbHRSSoFz3Bkn9uaRSk0UE/IsEBl0cQ= -github.com/projectdiscovery/retryabledns v1.0.56/go.mod h1:y6uQuDe9i2tcysDV7BTfKcTlfA3g0xyH5XITKBEw3as= +github.com/projectdiscovery/retryabledns v1.0.57 h1:+DOL9xYSIx74FRrOIKKHVp5R9ci53xHQN3jnncWVds4= +github.com/projectdiscovery/retryabledns v1.0.57/go.mod h1:qIigOcmO9d0Ce/z6mHzLl0Aiz2WJcNk2gUGhRcCQ1k4= github.com/projectdiscovery/retryablehttp-go v1.0.49 h1:mvlvl2kTN+ctpDIRlusVWui7eyFlElBoKTr8crS7yvY= github.com/projectdiscovery/retryablehttp-go v1.0.49/go.mod h1:VaJ7Au+1LP8C2u0qmx4NN1IdAxxkhoXpIcc9LAQzFo4= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= @@ -862,8 +862,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/useragent v0.0.39 h1:s2jyXdtjVo0MfYYkifx7irrOIoA0JhzhZaBkpcoWgV4= github.com/projectdiscovery/useragent v0.0.39/go.mod h1:wO6GQImJ2IQ5K+GDggS/Rhg6IV9Z2Du6NbqC/um0g0w= -github.com/projectdiscovery/utils v0.0.80 h1:daFuQwhVRtQ14JZs3DnI9ubaX273S8V1dZ+x/vr+YbI= -github.com/projectdiscovery/utils v0.0.80/go.mod h1:WXm3MIzKhgqUtTMwxDIW5bWe5nWkCYqRlZeqin0FqTc= +github.com/projectdiscovery/utils v0.0.83-0.20240305000020-ff30de2464cd h1:7AvCjcfZFkYy1Cg7aoA2iqVCuG/n+OHBFWB2ELGNAiI= +github.com/projectdiscovery/utils v0.0.83-0.20240305000020-ff30de2464cd/go.mod h1:67zb3eUa96XlTpWDqnJhe7xVoDbyAwHb7ChkdooiQxQ= github.com/projectdiscovery/wappalyzergo v0.0.111 h1:A1fLEycJ1zwvIVh3jCL1n5OaKn7KP+pHAsFqQYMXPZM= github.com/projectdiscovery/wappalyzergo v0.0.111/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -934,8 +934,10 @@ github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= -github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= +github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -1013,10 +1015,10 @@ github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaym github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= github.com/tim-ywliu/nested-logrus-formatter v1.3.2 h1:jugNJ2/CNCI79SxOJCOhwUHeN3O7/7/bj+ZRGOFlCSw= github.com/tim-ywliu/nested-logrus-formatter v1.3.2/go.mod h1:oGPmcxZB65j9Wo7mCnQKSrKEJtVDqyjD666SGmyStXI= -github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= -github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= -github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= -github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM= github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= @@ -1087,8 +1089,8 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os= github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= @@ -1360,6 +1362,7 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/pkg/operators/operators.go b/pkg/operators/operators.go index a3b4fc56..28045850 100644 --- a/pkg/operators/operators.go +++ b/pkg/operators/operators.go @@ -64,6 +64,22 @@ func (operators *Operators) Compile() error { return nil } +func (operators *Operators) HasDSL() bool { + for _, matcher := range operators.Matchers { + if len(matcher.DSL) > 0 { + return true + } + } + + for _, extractor := range operators.Extractors { + if len(extractor.DSL) > 0 { + return true + } + } + + return false +} + // GetMatchersCondition returns the condition for the matchers func (operators *Operators) GetMatchersCondition() matchers.ConditionType { return operators.matchersCondition diff --git a/pkg/output/output.go b/pkg/output/output.go index d2893cc3..7017325b 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -28,6 +28,7 @@ import ( protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/nuclei/v3/pkg/utils" + "github.com/projectdiscovery/nuclei/v3/pkg/utils/storage" fileutil "github.com/projectdiscovery/utils/file" osutils "github.com/projectdiscovery/utils/os" ) @@ -73,6 +74,19 @@ var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) // InternalEvent is an internal output generation structure for nuclei. type InternalEvent map[string]interface{} +func (ie InternalEvent) Set(k string, v interface{}) { + ie[k] = v +} + +func (ie InternalEvent) SetWithStorage(k string, v string, storage *storage.Storage) { + if hash, err := storage.SetString(v); err == nil { + ie[k] = hash + } else { + gologger.Print().Msgf("error: %s", err) + ie[k] = v + } +} + // InternalWrappedEvent is a wrapped event with operators result added to it. type InternalWrappedEvent struct { // Mutex is internal field which is implicitly used diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index f840934f..62590446 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "net/url" + "time" "github.com/go-sql-driver/mysql" "github.com/pkg/errors" @@ -15,16 +16,32 @@ import ( "github.com/projectdiscovery/networkpolicy" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/nuclei/v3/pkg/utils/expand" + "github.com/projectdiscovery/utils/memguardian" ) // Dialer is a shared fastdialer instance for host DNS resolution -var Dialer *fastdialer.Dialer +var ( + Dialer *fastdialer.Dialer + MemGuardian *memguardian.MemGuardian +) // Init creates the Dialer instance based on user configuration func Init(options *types.Options) error { if Dialer != nil { return nil } + if MemGuardian == nil { + mg, err := memguardian.New( + memguardian.WitInterval(time.Second), + memguardian.WithRatioWarning(75), + ) + if err != nil { + return err + } + MemGuardian = mg + go MemGuardian.Run(context.TODO()) + } + lfaAllowed = options.AllowLocalFileAccess opts := fastdialer.DefaultOptions if options.DialerTimeout > 0 { @@ -202,4 +219,7 @@ func Close() { if Dialer != nil { Dialer.Close() } + if MemGuardian != nil { + MemGuardian.Close() + } } diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go index cd88d304..cf92bc0f 100644 --- a/pkg/protocols/http/http.go +++ b/pkg/protocols/http/http.go @@ -3,6 +3,7 @@ package http import ( "bytes" "fmt" + "log" "strings" json "github.com/json-iterator/go" @@ -13,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/fuzz" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http" "github.com/projectdiscovery/rawhttp" @@ -388,8 +390,13 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } } if len(request.Payloads) > 0 { + // specifically for http requests high concurrency and and threads will lead to memory exausthion, hence reduce the maximum parallelism + if protocolstate.MemGuardian.Warning.Load() { + request.Threads = 5 + } // if we have payloads, adjust threads if none specified request.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads) + log.Println("request.Threads:", request.Threads) } return nil diff --git a/pkg/protocols/http/operators.go b/pkg/protocols/http/operators.go index c411ab82..3de3e63e 100644 --- a/pkg/protocols/http/operators.go +++ b/pkg/protocols/http/operators.go @@ -112,29 +112,21 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw data[k] = v } for _, cookie := range resp.Cookies() { - data[strings.ToLower(cookie.Name)] = cookie.Value + request.setHashOrDefault(data, strings.ToLower(cookie.Name), cookie.Value) } for k, v := range resp.Header { k = strings.ToLower(strings.ReplaceAll(strings.TrimSpace(k), "-", "_")) - data[k] = strings.Join(v, " ") + request.setHashOrDefault(data, k, strings.Join(v, " ")) } data["host"] = host data["type"] = request.Type().String() data["matched"] = matched - if hash, err := request.options.Storage.SetString(rawReq); err == nil { - data["request"] = hash - } else { - data["request"] = rawReq - } - if hash, err := request.options.Storage.SetString(rawResp); err == nil { - data["request"] = hash - } else { - data["response"] = rawResp - } + request.setHashOrDefault(data, "request", rawReq) + request.setHashOrDefault(data, "response", rawResp) data["status_code"] = resp.StatusCode - data["body"] = body - data["all_headers"] = headers - data["header"] = headers + request.setHashOrDefault(data, "body", body) + request.setHashOrDefault(data, "all_headers", headers) + request.setHashOrDefault(data, "header", headers) data["duration"] = duration.Seconds() data["template-id"] = request.options.TemplateID data["template-info"] = request.options.TemplateInfo @@ -148,6 +140,15 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw return data } +// TODO: disabling hdd storage while testing backpressure mechanism +func (request *Request) setHashOrDefault(data output.InternalEvent, k string, v string) { + // if hash, err := request.options.Storage.SetString(v); err == nil { + // data[k] = hash + // } else { + data[k] = v + //} +} + // MakeResultEvent creates a result event from internal wrapped event func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { return protocols.MakeDefaultResultEvent(request, wrapped) diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 508a3c5a..da0b995b 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -29,6 +29,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httputils" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signer" @@ -167,6 +168,10 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV // Workers that keeps enqueuing new requests maxWorkers := request.Threads + if protocolstate.MemGuardian.Warning.Load() { + maxWorkers = 5 + } + // Stop-at-first-match logic while executing requests // parallely using threads shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch) @@ -780,8 +785,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if request.CompiledOperators != nil { + event := &output.InternalWrappedEvent{} + if request.CompiledOperators != nil && request.CompiledOperators.HasDSL() { event.InternalEvent = outputEvent } callback(event) diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go index 380880b2..a5e310b7 100644 --- a/pkg/scan/scan_context.go +++ b/pkg/scan/scan_context.go @@ -10,6 +10,14 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" ) +type ScanContextOption func(*ScanContext) + +func WithEvents() ScanContextOption { + return func(sc *ScanContext) { + sc.withEvents = true + } +} + type ScanContext struct { context.Context // exported / configurable fields @@ -24,6 +32,10 @@ type ScanContext struct { errors []error warnings []string events []*output.InternalWrappedEvent + results []*output.ResultEvent + + // what to log + withEvents bool // might not be required but better to sync m sync.Mutex @@ -38,7 +50,8 @@ func NewScanContext(input *contextargs.Context) *ScanContext { func (s *ScanContext) GenerateResult() []*output.ResultEvent { s.m.Lock() defer s.m.Unlock() - return aggregateResults(s.events) + + return s.results } // LogEvent logs events to all events and triggeres any callbacks @@ -49,10 +62,16 @@ func (s *ScanContext) LogEvent(e *output.InternalWrappedEvent) { // do not log nil events return } + if s.OnResult != nil { s.OnResult(e) } - s.events = append(s.events, e) + + if s.withEvents { + s.events = append(s.events, e) + } + + s.results = append(s.results, e.Results...) } // LogError logs error to all events and triggeres any callbacks @@ -69,10 +88,11 @@ func (s *ScanContext) LogError(err error) { s.errors = append(s.errors, err) errorMessage := joinErrors(s.errors) - results := aggregateResults(s.events) - for _, result := range results { + + for _, result := range s.results { result.Error = errorMessage } + for _, e := range s.events { e.InternalEvent["error"] = errorMessage } @@ -97,15 +117,6 @@ func (s *ScanContext) LogWarning(format string, args ...any) { } } -// aggregateResults aggregates results from multiple events -func aggregateResults(events []*output.InternalWrappedEvent) []*output.ResultEvent { - var results []*output.ResultEvent - for _, e := range events { - results = append(results, e.Results...) - } - return results -} - // joinErrors joins multiple errors and returns a single error string func joinErrors(errors []error) string { var errorMessages []string diff --git a/pkg/utils/storage/storage.go b/pkg/utils/storage/storage.go index bca06730..2e0772a6 100644 --- a/pkg/utils/storage/storage.go +++ b/pkg/utils/storage/storage.go @@ -1,10 +1,10 @@ package storage import ( - "crypto/sha1" - "encoding/hex" + "fmt" "os" + "github.com/cespare/xxhash/v2" "github.com/projectdiscovery/utils/conversion" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/opt" @@ -36,35 +36,26 @@ func (s *Storage) Close() { os.RemoveAll(s.dbPath) } -func Hash(v []byte) []byte { - hasher := sha1.New() - _, _ = hasher.Write(v) - return hasher.Sum(nil) +func HashString(v string) uint64 { + return Hash(conversion.Bytes(v)) } -func HashString(v []byte) string { - return hex.EncodeToString(v) +func Hash(v []byte) uint64 { + return xxhash.Sum64(v) } -func HashBytes(v string) []byte { - hash, _ := hex.DecodeString(v) - return hash -} - -func (s *Storage) Get(k string) (string, error) { - hash := HashBytes(k) - - v, err := s.storage.Get(hash, nil) +func (s *Storage) Get(k uint64) (string, error) { + v, err := s.storage.Get(conversion.Bytes(fmt.Sprint(k)), nil) return conversion.String(v), err } -func (s *Storage) SetString(v string) (string, error) { +func (s *Storage) SetString(v string) (uint64, error) { return s.Set(conversion.Bytes(v)) } -func (s *Storage) Set(v []byte) (string, error) { +func (s *Storage) Set(v []byte) (uint64, error) { hash := Hash(v) - return HashString(hash), s.storage.Put(hash, v, nil) + return hash, s.storage.Put(conversion.Bytes(fmt.Sprint(hash)), v, nil) }