mirror of https://github.com/daffainfo/nuclei.git
adding disk storage + memguardian + other memory optimizations
mzack
parent
4d29bff759
commit
ea3ecbf2d2
19
go.mod
19
go.mod
|
|
@ -21,16 +21,16 @@ require (
|
|||
github.com/pkg/errors v0.9.1
|
||||
github.com/projectdiscovery/clistats v0.0.20
|
||||
github.com/projectdiscovery/fastdialer v0.0.60
|
||||
github.com/projectdiscovery/hmap v0.0.39
|
||||
github.com/projectdiscovery/hmap v0.0.40
|
||||
github.com/projectdiscovery/interactsh v1.1.8
|
||||
github.com/projectdiscovery/rawhttp v0.1.35
|
||||
github.com/projectdiscovery/retryabledns v1.0.56
|
||||
github.com/projectdiscovery/retryabledns v1.0.57
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.49
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.4
|
||||
github.com/remeh/sizedwaitgroup v1.0.0
|
||||
github.com/rs/xid v1.5.0
|
||||
github.com/segmentio/ksuid v1.0.4
|
||||
github.com/shirou/gopsutil/v3 v3.23.7 // indirect
|
||||
github.com/shirou/gopsutil/v3 v3.24.2 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/spf13/cast v1.5.1
|
||||
github.com/syndtr/goleveldb v1.0.0
|
||||
|
|
@ -59,6 +59,7 @@ require (
|
|||
github.com/aws/aws-sdk-go-v2/credentials v1.13.27
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0
|
||||
github.com/cespare/xxhash/v2 v2.2.0
|
||||
github.com/charmbracelet/glamour v0.6.0
|
||||
github.com/denisenkom/go-mssqldb v0.12.3
|
||||
github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c
|
||||
|
|
@ -91,7 +92,7 @@ require (
|
|||
github.com/projectdiscovery/tlsx v1.1.6
|
||||
github.com/projectdiscovery/uncover v1.0.7
|
||||
github.com/projectdiscovery/useragent v0.0.39
|
||||
github.com/projectdiscovery/utils v0.0.80
|
||||
github.com/projectdiscovery/utils v0.0.83-0.20240305000020-ff30de2464cd
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.111
|
||||
github.com/redis/go-redis/v9 v9.1.0
|
||||
github.com/sashabaranov/go-openai v1.15.3
|
||||
|
|
@ -123,7 +124,6 @@ require (
|
|||
github.com/bits-and-blooms/bloom/v3 v3.5.0 // indirect
|
||||
github.com/bytedance/sonic v1.9.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/cheggaaa/pb/v3 v3.1.4 // indirect
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
|
||||
github.com/cloudflare/cfssl v1.6.4 // indirect
|
||||
|
|
@ -132,7 +132,6 @@ require (
|
|||
github.com/corpix/uarand v0.2.0 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
|
||||
github.com/davidmz/go-pageant v1.0.2 // indirect
|
||||
github.com/denisbrodbeck/machineid v1.0.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dlclark/regexp2 v1.10.0 // indirect
|
||||
github.com/docker/cli v24.0.5+incompatible // indirect
|
||||
|
|
@ -188,9 +187,11 @@ require (
|
|||
github.com/projectdiscovery/asnmap v1.0.6 // indirect
|
||||
github.com/projectdiscovery/cdncheck v1.0.9 // indirect
|
||||
github.com/projectdiscovery/freeport v0.0.5 // indirect
|
||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
|
||||
github.com/projectdiscovery/stringsutil v0.0.2 // indirect
|
||||
github.com/quic-go/quic-go v0.40.1 // indirect
|
||||
github.com/refraction-networking/utls v1.6.1 // indirect
|
||||
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
|
||||
github.com/shoenig/go-m1cpu v0.1.6 // indirect
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
github.com/skeema/knownhosts v1.2.1 // indirect
|
||||
|
|
@ -270,8 +271,8 @@ require (
|
|||
github.com/projectdiscovery/networkpolicy v0.0.7
|
||||
github.com/rivo/uniseg v0.4.4 // indirect
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.11 // indirect
|
||||
github.com/tklauser/numcpus v0.6.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.12 // indirect
|
||||
github.com/tklauser/numcpus v0.6.1 // indirect
|
||||
github.com/trivago/tgo v1.0.7
|
||||
github.com/ulikunitz/xz v0.5.11 // indirect
|
||||
github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
|
||||
|
|
@ -280,7 +281,7 @@ require (
|
|||
github.com/ysmood/goob v0.4.0 // indirect
|
||||
github.com/ysmood/gson v0.7.3 // indirect
|
||||
github.com/ysmood/leakless v0.8.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.3 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
|
||||
github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c // indirect
|
||||
go.etcd.io/bbolt v1.3.8 // indirect
|
||||
|
|
|
|||
35
go.sum
35
go.sum
|
|
@ -260,8 +260,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
|||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0=
|
||||
github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE=
|
||||
github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
|
||||
github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
|
||||
github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw=
|
||||
github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
|
|
@ -830,12 +828,14 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
|
|||
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
|
||||
github.com/projectdiscovery/gozero v0.0.1 h1:f08ZnYlbDZV/TNGDvIXV9s/oB/sAI+HWaSbW4em4aKM=
|
||||
github.com/projectdiscovery/gozero v0.0.1/go.mod h1:/dHwbly+1lhOX9UreVure4lEe7K4hIHeu/c/wZGNTDo=
|
||||
github.com/projectdiscovery/hmap v0.0.39 h1:R33JJzPz8TMUm1TJQ/X5zVJbmyTS64EttZ085thq19g=
|
||||
github.com/projectdiscovery/hmap v0.0.39/go.mod h1:wEPoEIVGPdHsc9EnE+ERUdgNyp29zZn6gACOALylHOg=
|
||||
github.com/projectdiscovery/hmap v0.0.40 h1:WGAIXXMY2vbV0ep7Q8s27Up/ejs8Wo1hh5AEhynLfmw=
|
||||
github.com/projectdiscovery/hmap v0.0.40/go.mod h1:5JkQW9t/UNK95YEY6irOXHqrS/xVBUtrYEhtq61ttII=
|
||||
github.com/projectdiscovery/httpx v1.3.9 h1:jDdoGH+5VVU/jI6dnai1DKNw9USPyCcw+tDh4RCVQ2g=
|
||||
github.com/projectdiscovery/httpx v1.3.9/go.mod h1:a/a5X6e2NLnS/+b3buFadGUpZSolnVkMA7KZdpCdg58=
|
||||
github.com/projectdiscovery/interactsh v1.1.8 h1:mDD+f/oo2tV4Z1WyUync0tgYeJyuiS89Un64Gm6Pvgk=
|
||||
github.com/projectdiscovery/interactsh v1.1.8/go.mod h1:E20ywFb7bL01GcOOk+6VZF48XZ8AZvYvBpULoBUSTbg=
|
||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
|
||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
|
||||
github.com/projectdiscovery/mapcidr v1.1.16 h1:rjj1w5D6hbTsUQXYClLcGdfBEy9bryclgi70t0vBggo=
|
||||
github.com/projectdiscovery/mapcidr v1.1.16/go.mod h1:rGqpBhStdwOQ2uS62QM9qPsybwMwIhT7CTd2bxoHs8Q=
|
||||
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
|
||||
|
|
@ -848,8 +848,8 @@ github.com/projectdiscovery/rawhttp v0.1.35 h1:9Hkbu1WLN5coj6+HBaqi26PjMNFnw1XrM
|
|||
github.com/projectdiscovery/rawhttp v0.1.35/go.mod h1:9mS0N3BfOBYwQWgyI+bXBaFVMFBtJVTcZF0FENea7mA=
|
||||
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
|
||||
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
|
||||
github.com/projectdiscovery/retryabledns v1.0.56 h1:Rk/fvBSNjw4vzbHRSSoFz3Bkn9uaRSk0UE/IsEBl0cQ=
|
||||
github.com/projectdiscovery/retryabledns v1.0.56/go.mod h1:y6uQuDe9i2tcysDV7BTfKcTlfA3g0xyH5XITKBEw3as=
|
||||
github.com/projectdiscovery/retryabledns v1.0.57 h1:+DOL9xYSIx74FRrOIKKHVp5R9ci53xHQN3jnncWVds4=
|
||||
github.com/projectdiscovery/retryabledns v1.0.57/go.mod h1:qIigOcmO9d0Ce/z6mHzLl0Aiz2WJcNk2gUGhRcCQ1k4=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.49 h1:mvlvl2kTN+ctpDIRlusVWui7eyFlElBoKTr8crS7yvY=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.49/go.mod h1:VaJ7Au+1LP8C2u0qmx4NN1IdAxxkhoXpIcc9LAQzFo4=
|
||||
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
|
||||
|
|
@ -862,8 +862,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
|
|||
github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
|
||||
github.com/projectdiscovery/useragent v0.0.39 h1:s2jyXdtjVo0MfYYkifx7irrOIoA0JhzhZaBkpcoWgV4=
|
||||
github.com/projectdiscovery/useragent v0.0.39/go.mod h1:wO6GQImJ2IQ5K+GDggS/Rhg6IV9Z2Du6NbqC/um0g0w=
|
||||
github.com/projectdiscovery/utils v0.0.80 h1:daFuQwhVRtQ14JZs3DnI9ubaX273S8V1dZ+x/vr+YbI=
|
||||
github.com/projectdiscovery/utils v0.0.80/go.mod h1:WXm3MIzKhgqUtTMwxDIW5bWe5nWkCYqRlZeqin0FqTc=
|
||||
github.com/projectdiscovery/utils v0.0.83-0.20240305000020-ff30de2464cd h1:7AvCjcfZFkYy1Cg7aoA2iqVCuG/n+OHBFWB2ELGNAiI=
|
||||
github.com/projectdiscovery/utils v0.0.83-0.20240305000020-ff30de2464cd/go.mod h1:67zb3eUa96XlTpWDqnJhe7xVoDbyAwHb7ChkdooiQxQ=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.111 h1:A1fLEycJ1zwvIVh3jCL1n5OaKn7KP+pHAsFqQYMXPZM=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.111/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
|
||||
|
|
@ -934,8 +934,10 @@ github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O
|
|||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
|
||||
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4=
|
||||
github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4=
|
||||
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
|
||||
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y=
|
||||
github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk=
|
||||
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
|
||||
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
|
||||
github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
|
||||
|
|
@ -1013,10 +1015,10 @@ github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaym
|
|||
github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw=
|
||||
github.com/tim-ywliu/nested-logrus-formatter v1.3.2 h1:jugNJ2/CNCI79SxOJCOhwUHeN3O7/7/bj+ZRGOFlCSw=
|
||||
github.com/tim-ywliu/nested-logrus-formatter v1.3.2/go.mod h1:oGPmcxZB65j9Wo7mCnQKSrKEJtVDqyjD666SGmyStXI=
|
||||
github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
|
||||
github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
|
||||
github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
|
||||
github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
|
||||
github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
|
||||
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
|
||||
github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
|
||||
github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM=
|
||||
github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
|
||||
|
|
@ -1087,8 +1089,8 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU=
|
|||
github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os=
|
||||
github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ=
|
||||
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
|
||||
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
|
||||
github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
|
||||
|
|
@ -1360,6 +1362,7 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
|||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
|
|
|
|||
|
|
@ -64,6 +64,22 @@ func (operators *Operators) Compile() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (operators *Operators) HasDSL() bool {
|
||||
for _, matcher := range operators.Matchers {
|
||||
if len(matcher.DSL) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
for _, extractor := range operators.Extractors {
|
||||
if len(extractor.DSL) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// GetMatchersCondition returns the condition for the matchers
|
||||
func (operators *Operators) GetMatchersCondition() matchers.ConditionType {
|
||||
return operators.matchersCondition
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import (
|
|||
protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/storage"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
osutils "github.com/projectdiscovery/utils/os"
|
||||
)
|
||||
|
|
@ -73,6 +74,19 @@ var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
|
|||
// InternalEvent is an internal output generation structure for nuclei.
|
||||
type InternalEvent map[string]interface{}
|
||||
|
||||
func (ie InternalEvent) Set(k string, v interface{}) {
|
||||
ie[k] = v
|
||||
}
|
||||
|
||||
func (ie InternalEvent) SetWithStorage(k string, v string, storage *storage.Storage) {
|
||||
if hash, err := storage.SetString(v); err == nil {
|
||||
ie[k] = hash
|
||||
} else {
|
||||
gologger.Print().Msgf("error: %s", err)
|
||||
ie[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
// InternalWrappedEvent is a wrapped event with operators result added to it.
|
||||
type InternalWrappedEvent struct {
|
||||
// Mutex is internal field which is implicitly used
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
"github.com/pkg/errors"
|
||||
|
|
@ -15,16 +16,32 @@ import (
|
|||
"github.com/projectdiscovery/networkpolicy"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/expand"
|
||||
"github.com/projectdiscovery/utils/memguardian"
|
||||
)
|
||||
|
||||
// Dialer is a shared fastdialer instance for host DNS resolution
|
||||
var Dialer *fastdialer.Dialer
|
||||
var (
|
||||
Dialer *fastdialer.Dialer
|
||||
MemGuardian *memguardian.MemGuardian
|
||||
)
|
||||
|
||||
// Init creates the Dialer instance based on user configuration
|
||||
func Init(options *types.Options) error {
|
||||
if Dialer != nil {
|
||||
return nil
|
||||
}
|
||||
if MemGuardian == nil {
|
||||
mg, err := memguardian.New(
|
||||
memguardian.WitInterval(time.Second),
|
||||
memguardian.WithRatioWarning(75),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
MemGuardian = mg
|
||||
go MemGuardian.Run(context.TODO())
|
||||
}
|
||||
|
||||
lfaAllowed = options.AllowLocalFileAccess
|
||||
opts := fastdialer.DefaultOptions
|
||||
if options.DialerTimeout > 0 {
|
||||
|
|
@ -202,4 +219,7 @@ func Close() {
|
|||
if Dialer != nil {
|
||||
Dialer.Close()
|
||||
}
|
||||
if MemGuardian != nil {
|
||||
MemGuardian.Close()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package http
|
|||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
json "github.com/json-iterator/go"
|
||||
|
|
@ -13,6 +14,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/fuzz"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
|
||||
"github.com/projectdiscovery/rawhttp"
|
||||
|
|
@ -388,8 +390,13 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
|
|||
}
|
||||
}
|
||||
if len(request.Payloads) > 0 {
|
||||
// specifically for http requests high concurrency and and threads will lead to memory exausthion, hence reduce the maximum parallelism
|
||||
if protocolstate.MemGuardian.Warning.Load() {
|
||||
request.Threads = 5
|
||||
}
|
||||
// if we have payloads, adjust threads if none specified
|
||||
request.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads)
|
||||
log.Println("request.Threads:", request.Threads)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -112,29 +112,21 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw
|
|||
data[k] = v
|
||||
}
|
||||
for _, cookie := range resp.Cookies() {
|
||||
data[strings.ToLower(cookie.Name)] = cookie.Value
|
||||
request.setHashOrDefault(data, strings.ToLower(cookie.Name), cookie.Value)
|
||||
}
|
||||
for k, v := range resp.Header {
|
||||
k = strings.ToLower(strings.ReplaceAll(strings.TrimSpace(k), "-", "_"))
|
||||
data[k] = strings.Join(v, " ")
|
||||
request.setHashOrDefault(data, k, strings.Join(v, " "))
|
||||
}
|
||||
data["host"] = host
|
||||
data["type"] = request.Type().String()
|
||||
data["matched"] = matched
|
||||
if hash, err := request.options.Storage.SetString(rawReq); err == nil {
|
||||
data["request"] = hash
|
||||
} else {
|
||||
data["request"] = rawReq
|
||||
}
|
||||
if hash, err := request.options.Storage.SetString(rawResp); err == nil {
|
||||
data["request"] = hash
|
||||
} else {
|
||||
data["response"] = rawResp
|
||||
}
|
||||
request.setHashOrDefault(data, "request", rawReq)
|
||||
request.setHashOrDefault(data, "response", rawResp)
|
||||
data["status_code"] = resp.StatusCode
|
||||
data["body"] = body
|
||||
data["all_headers"] = headers
|
||||
data["header"] = headers
|
||||
request.setHashOrDefault(data, "body", body)
|
||||
request.setHashOrDefault(data, "all_headers", headers)
|
||||
request.setHashOrDefault(data, "header", headers)
|
||||
data["duration"] = duration.Seconds()
|
||||
data["template-id"] = request.options.TemplateID
|
||||
data["template-info"] = request.options.TemplateInfo
|
||||
|
|
@ -148,6 +140,15 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw
|
|||
return data
|
||||
}
|
||||
|
||||
// TODO: disabling hdd storage while testing backpressure mechanism
|
||||
func (request *Request) setHashOrDefault(data output.InternalEvent, k string, v string) {
|
||||
// if hash, err := request.options.Storage.SetString(v); err == nil {
|
||||
// data[k] = hash
|
||||
// } else {
|
||||
data[k] = v
|
||||
//}
|
||||
}
|
||||
|
||||
// MakeResultEvent creates a result event from internal wrapped event
|
||||
func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
|
||||
return protocols.MakeDefaultResultEvent(request, wrapped)
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httputils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signer"
|
||||
|
|
@ -167,6 +168,10 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
|
|||
// Workers that keeps enqueuing new requests
|
||||
maxWorkers := request.Threads
|
||||
|
||||
if protocolstate.MemGuardian.Warning.Load() {
|
||||
maxWorkers = 5
|
||||
}
|
||||
|
||||
// Stop-at-first-match logic while executing requests
|
||||
// parallely using threads
|
||||
shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch)
|
||||
|
|
@ -780,8 +785,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
|
||||
}
|
||||
|
||||
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
|
||||
if request.CompiledOperators != nil {
|
||||
event := &output.InternalWrappedEvent{}
|
||||
if request.CompiledOperators != nil && request.CompiledOperators.HasDSL() {
|
||||
event.InternalEvent = outputEvent
|
||||
}
|
||||
callback(event)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,14 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
)
|
||||
|
||||
type ScanContextOption func(*ScanContext)
|
||||
|
||||
func WithEvents() ScanContextOption {
|
||||
return func(sc *ScanContext) {
|
||||
sc.withEvents = true
|
||||
}
|
||||
}
|
||||
|
||||
type ScanContext struct {
|
||||
context.Context
|
||||
// exported / configurable fields
|
||||
|
|
@ -24,6 +32,10 @@ type ScanContext struct {
|
|||
errors []error
|
||||
warnings []string
|
||||
events []*output.InternalWrappedEvent
|
||||
results []*output.ResultEvent
|
||||
|
||||
// what to log
|
||||
withEvents bool
|
||||
|
||||
// might not be required but better to sync
|
||||
m sync.Mutex
|
||||
|
|
@ -38,7 +50,8 @@ func NewScanContext(input *contextargs.Context) *ScanContext {
|
|||
func (s *ScanContext) GenerateResult() []*output.ResultEvent {
|
||||
s.m.Lock()
|
||||
defer s.m.Unlock()
|
||||
return aggregateResults(s.events)
|
||||
|
||||
return s.results
|
||||
}
|
||||
|
||||
// LogEvent logs events to all events and triggeres any callbacks
|
||||
|
|
@ -49,10 +62,16 @@ func (s *ScanContext) LogEvent(e *output.InternalWrappedEvent) {
|
|||
// do not log nil events
|
||||
return
|
||||
}
|
||||
|
||||
if s.OnResult != nil {
|
||||
s.OnResult(e)
|
||||
}
|
||||
|
||||
if s.withEvents {
|
||||
s.events = append(s.events, e)
|
||||
}
|
||||
|
||||
s.results = append(s.results, e.Results...)
|
||||
}
|
||||
|
||||
// LogError logs error to all events and triggeres any callbacks
|
||||
|
|
@ -69,10 +88,11 @@ func (s *ScanContext) LogError(err error) {
|
|||
s.errors = append(s.errors, err)
|
||||
|
||||
errorMessage := joinErrors(s.errors)
|
||||
results := aggregateResults(s.events)
|
||||
for _, result := range results {
|
||||
|
||||
for _, result := range s.results {
|
||||
result.Error = errorMessage
|
||||
}
|
||||
|
||||
for _, e := range s.events {
|
||||
e.InternalEvent["error"] = errorMessage
|
||||
}
|
||||
|
|
@ -97,15 +117,6 @@ func (s *ScanContext) LogWarning(format string, args ...any) {
|
|||
}
|
||||
}
|
||||
|
||||
// aggregateResults aggregates results from multiple events
|
||||
func aggregateResults(events []*output.InternalWrappedEvent) []*output.ResultEvent {
|
||||
var results []*output.ResultEvent
|
||||
for _, e := range events {
|
||||
results = append(results, e.Results...)
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
// joinErrors joins multiple errors and returns a single error string
|
||||
func joinErrors(errors []error) string {
|
||||
var errorMessages []string
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
package storage
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
"github.com/projectdiscovery/utils/conversion"
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
|
|
@ -36,35 +36,26 @@ func (s *Storage) Close() {
|
|||
os.RemoveAll(s.dbPath)
|
||||
}
|
||||
|
||||
func Hash(v []byte) []byte {
|
||||
hasher := sha1.New()
|
||||
_, _ = hasher.Write(v)
|
||||
return hasher.Sum(nil)
|
||||
func HashString(v string) uint64 {
|
||||
return Hash(conversion.Bytes(v))
|
||||
}
|
||||
|
||||
func HashString(v []byte) string {
|
||||
return hex.EncodeToString(v)
|
||||
func Hash(v []byte) uint64 {
|
||||
return xxhash.Sum64(v)
|
||||
}
|
||||
|
||||
func HashBytes(v string) []byte {
|
||||
hash, _ := hex.DecodeString(v)
|
||||
return hash
|
||||
}
|
||||
|
||||
func (s *Storage) Get(k string) (string, error) {
|
||||
hash := HashBytes(k)
|
||||
|
||||
v, err := s.storage.Get(hash, nil)
|
||||
func (s *Storage) Get(k uint64) (string, error) {
|
||||
v, err := s.storage.Get(conversion.Bytes(fmt.Sprint(k)), nil)
|
||||
|
||||
return conversion.String(v), err
|
||||
}
|
||||
|
||||
func (s *Storage) SetString(v string) (string, error) {
|
||||
func (s *Storage) SetString(v string) (uint64, error) {
|
||||
return s.Set(conversion.Bytes(v))
|
||||
}
|
||||
|
||||
func (s *Storage) Set(v []byte) (string, error) {
|
||||
func (s *Storage) Set(v []byte) (uint64, error) {
|
||||
hash := Hash(v)
|
||||
|
||||
return HashString(hash), s.storage.Put(hash, v, nil)
|
||||
return hash, s.storage.Put(conversion.Bytes(fmt.Sprint(hash)), v, nil)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue