Merge pull request #405 from projectdiscovery/bugfix-clistats

Hmap + clistats
dev
bauthard 2020-11-19 12:13:33 +05:30 committed by GitHub
commit 9e6e6d3267
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 154 additions and 250 deletions

View File

@ -6,29 +6,24 @@ require (
github.com/Knetic/govaluate v3.0.0+incompatible
github.com/blang/semver v3.5.1+incompatible
github.com/d5/tengo/v2 v2.6.2
github.com/golang/snappy v0.0.2 // indirect
github.com/google/go-github/v32 v32.1.0
github.com/json-iterator/go v1.1.10
github.com/karrick/godirwalk v1.16.1
github.com/logrusorgru/aurora v2.0.3+incompatible
github.com/miekg/dns v1.1.35
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.4
github.com/projectdiscovery/collaborator v0.0.0-20201106184551-efd6773a9479
github.com/projectdiscovery/clistats v0.0.5
github.com/projectdiscovery/collaborator v0.0.1
github.com/projectdiscovery/fastdialer v0.0.1
github.com/projectdiscovery/gologger v1.0.1
github.com/projectdiscovery/hmap v0.0.1
github.com/projectdiscovery/rawhttp v0.0.5-0.20201030111302-fd184be37926
github.com/projectdiscovery/rawhttp v0.0.4
github.com/projectdiscovery/retryabledns v1.0.5
github.com/projectdiscovery/retryablehttp-go v1.0.1
github.com/remeh/sizedwaitgroup v1.0.0
github.com/segmentio/ksuid v1.0.3
github.com/spaolacci/murmur3 v1.1.0
github.com/stretchr/testify v1.5.1
go.uber.org/ratelimit v0.1.0
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 // indirect
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102
golang.org/x/sys v0.0.0-20201106081118-db71ae66460a // indirect
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
gopkg.in/yaml.v2 v2.3.0
)

View File

@ -1,30 +1,26 @@
github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4=
github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc=
github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/eiannone/keyboard v0.0.0-20200508000154-caf4b762e807 h1:jdjd5e68T4R/j4PWxfZqcKY8KtT9oo8IPNVuV4bSXDQ=
github.com/eiannone/keyboard v0.0.0-20200508000154-caf4b762e807/go.mod h1:Xoiu5VdKMvbRgHuY7+z64lhu/7lvax/22nzASF6GrO8=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@ -34,63 +30,42 @@ github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQ
github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.34 h1:SgTzfkN+oLoIHF1bgUP+C71mzuDl3AhLApHzCCIAMWM=
github.com/miekg/dns v1.1.34/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs=
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/projectdiscovery/clistats v0.0.4 h1:YdxLQAVM35ZQkDquz8yG9R1aOXDBYeEVsgeqzUm4XXc=
github.com/projectdiscovery/clistats v0.0.4/go.mod h1:+RpHbYUY4FJ6yQiDlL4Ux44jK3p4AwFKUsr6neu43RA=
github.com/projectdiscovery/collaborator v0.0.0-20201023080839-2aa1290ed09d h1:iHb2v6VX1Fjl9IX8UaJFLKLej+KyJJ59W1pWIRCug4Q=
github.com/projectdiscovery/collaborator v0.0.0-20201023080839-2aa1290ed09d/go.mod h1:M7Csn+hQVDOLCEEFkj6dazmtgG1tIqJpbuPHlRlpYGQ=
github.com/projectdiscovery/collaborator v0.0.0-20201106184551-efd6773a9479 h1:2K1Vg9jh6bX5fBz05y45VGE2K3TgPXOkGu/EevCbrBs=
github.com/projectdiscovery/collaborator v0.0.0-20201106184551-efd6773a9479/go.mod h1:pT+nfywyhy3cJacTU6PLUbE/uHqabzoN4xp4DcF9qoY=
github.com/projectdiscovery/fastdialer v0.0.0-20201106202608-63bccd73532b h1:O4vf9HH4DPhAg3sUzXyNqejHJbXwdMyHFnwzVNLZqBY=
github.com/projectdiscovery/fastdialer v0.0.0-20201106202608-63bccd73532b/go.mod h1:TIWLavgdJqcDF0K+y/wfW/0MXA8ieNPzLPgr3cFkxU8=
github.com/projectdiscovery/fastdialer v0.0.0-20201109221154-72c33a5f6a74 h1:6y/HAJOSQ99URPRiB9xQ1/3xszxEyjDEy+m5zUq0S/U=
github.com/projectdiscovery/fastdialer v0.0.0-20201109221154-72c33a5f6a74/go.mod h1:TIWLavgdJqcDF0K+y/wfW/0MXA8ieNPzLPgr3cFkxU8=
github.com/projectdiscovery/clistats v0.0.5 h1:vcvOR9PrFRawO/7FWD6pER9nYVSoSTD2F+/fkRs73a0=
github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
github.com/projectdiscovery/collaborator v0.0.1 h1:dbQ5BCL/a3c+BB9cGtrGgiLs23+EfSzoaTzX/pxqiTI=
github.com/projectdiscovery/collaborator v0.0.1/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
github.com/projectdiscovery/fastdialer v0.0.1 h1:MgBkJ/zkciFu/PcbAz0DYGiZn2aqv6b39NvfXxfN8qg=
github.com/projectdiscovery/fastdialer v0.0.1/go.mod h1:d24GUzSb93wOY7lu4gJmXAzfomqAGEcRrInEVrM6zbc=
github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ=
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
github.com/projectdiscovery/hmap v0.0.0-20201026185329-db41b5717bcb h1:y+O2ZCGICrGcSevxu2nydZtjMiwDHPf2cMgwi8PCFgQ=
github.com/projectdiscovery/hmap v0.0.0-20201026185329-db41b5717bcb/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
github.com/projectdiscovery/rawhttp v0.0.5-0.20201030111302-fd184be37926 h1:aAh1EqrurioC8OeOJAzZG2gYN7MEmMNc2UWLJnQNV78=
github.com/projectdiscovery/rawhttp v0.0.5-0.20201030111302-fd184be37926/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60=
github.com/projectdiscovery/retryabledns v1.0.4/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
github.com/projectdiscovery/retryabledns v1.0.5-0.20201109220805-cead885aba17 h1:70d4IzHGYEHFOh4qLCb0a9MbpWR7zXOMI1BGgz1SgyU=
github.com/projectdiscovery/retryabledns v1.0.5-0.20201109220805-cead885aba17/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
github.com/projectdiscovery/rawhttp v0.0.4 h1:O5IreNGk83d4xTD9e6SpkKbX0sHTs8K1Q33Bz4eYl2E=
github.com/projectdiscovery/rawhttp v0.0.4/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/retryabledns v1.0.5 h1:bQivGy5CuqKlwcxRkgA5ENincqIed/BR2sA6t2gdwuI=
github.com/projectdiscovery/retryabledns v1.0.5/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY=
github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
@ -101,8 +76,8 @@ go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 h1:pLI5jrR7OSLijeIDcmRxNmw2api+jEfxLoykJVice/E=
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9 h1:umElSU9WZirRdgu2yFHY0ayQkEnKiOC1TtM3fWXFnoU=
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -110,37 +85,27 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201029221708-28c70e62bb1d h1:dOiJ2n2cMwGLce/74I/QHMbnpk5GfY7InR8rczoMqRM=
golang.org/x/net v0.0.0-20201029221708-28c70e62bb1d/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102 h1:42cLlJJdEh+ySyeUUbEQ5bsTiq8voBeTuweGVkY6Puw=
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 h1:HlFl4V6pEMziuLXyRkm5BIYq1y1GAbb02pRlWvI54OM=
golang.org/x/sys v0.0.0-20201029080932-201ba4db2418/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201106081118-db71ae66460a h1:ALUFBKlIyeY7y5ZgPJmblk/vKz+zBQSnNiPkt41sgeg=
golang.org/x/sys v0.0.0-20201106081118-db71ae66460a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba h1:xmhUJGQGbxlod18iJGqVEp9cHIPLl7QiX2aA3to708s=
golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

View File

@ -2,6 +2,7 @@ package progress
import (
"fmt"
"os"
"strings"
"time"
@ -11,6 +12,7 @@ import (
// Progress is a progress instance for showing program stats
type Progress struct {
active bool
stats clistats.StatisticsClient
tickDuration time.Duration
}
@ -19,45 +21,61 @@ type Progress struct {
func NewProgress(active bool) *Progress {
var tickDuration time.Duration
if active {
tickDuration = 1 * time.Second
tickDuration = 5 * time.Second
} else {
tickDuration = -1
}
progress := &Progress{
tickDuration: tickDuration,
stats: clistats.New(),
var progress Progress
if active {
stats, err := clistats.New()
if err != nil {
gologger.Warningf("Couldn't create progress engine: %s\n", err)
}
progress.active = active
progress.stats = stats
progress.tickDuration = tickDuration
}
return progress
return &progress
}
// Init initializes the progress display mechanism by setting counters, etc.
func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) {
p.stats.AddStatic("templates", rulesCount)
p.stats.AddStatic("hosts", hostCount)
p.stats.AddStatic("startedAt", time.Now())
p.stats.AddCounter("requests", uint64(0))
p.stats.AddCounter("errors", uint64(0))
p.stats.AddCounter("total", uint64(requestCount))
_ = p.stats.Start(makePrintCallback(), p.tickDuration)
if p.active {
p.stats.AddStatic("templates", rulesCount)
p.stats.AddStatic("hosts", hostCount)
p.stats.AddStatic("startedAt", time.Now())
p.stats.AddCounter("requests", uint64(0))
p.stats.AddCounter("errors", uint64(0))
p.stats.AddCounter("total", uint64(requestCount))
if err := p.stats.Start(makePrintCallback(), p.tickDuration); err != nil {
gologger.Warningf("Couldn't start statistics: %s\n", err)
}
}
}
// AddToTotal adds a value to the total request count
func (p *Progress) AddToTotal(delta int64) {
p.stats.IncrementCounter("total", int(delta))
if p.active {
p.stats.IncrementCounter("total", int(delta))
}
}
// Update progress tracking information and increments the request counter by one unit.
func (p *Progress) Update() {
p.stats.IncrementCounter("requests", 1)
if p.active {
p.stats.IncrementCounter("requests", 1)
}
}
// Drop drops the specified number of requests from the progress bar total.
// This may be the case when uncompleted requests are encountered and shouldn't be part of the total count.
func (p *Progress) Drop(count int64) {
// mimic dropping by incrementing the completed requests
p.stats.IncrementCounter("errors", int(count))
if p.active {
// mimic dropping by incrementing the completed requests
p.stats.IncrementCounter("errors", int(count))
}
}
const bufferSize = 128
@ -102,7 +120,7 @@ func makePrintCallback() func(stats clistats.StatisticsClient) {
builder.WriteRune(')')
builder.WriteRune('\n')
gologger.Printf("%s", builder.String())
fmt.Fprintf(os.Stderr, "%s", builder.String())
builder.Reset()
}
}
@ -120,5 +138,9 @@ func fmtDuration(d time.Duration) string {
// Stop stops the progress bar execution
func (p *Progress) Stop() {
_ = p.stats.Stop()
if p.active {
if err := p.stats.Stop(); err != nil {
gologger.Warningf("Couldn't stop statistics: %s\n", err)
}
}
}

View File

@ -88,7 +88,7 @@ func ParseOptions() *Options {
flag.BoolVar(&options.JSONRequests, "json-requests", false, "Write requests/responses for matches in JSON output")
flag.BoolVar(&options.EnableProgressBar, "pbar", false, "Enable the progress bar")
flag.BoolVar(&options.TemplateList, "tl", false, "List available templates")
flag.IntVar(&options.RateLimit, "rate-limit", 150, "Rate-Limit Per Target (maximum requests/second")
flag.IntVar(&options.RateLimit, "rate-limit", 150, "Rate-Limit (maximum requests/second")
flag.BoolVar(&options.StopAtFirstMatch, "stop-at-first-match", false, "Stop processing http requests at first match (this may break template/workflow logic)")
flag.IntVar(&options.BulkSize, "bulk-size", 25, "Maximum Number of hosts analyzed in parallel per template")
flag.IntVar(&options.TemplateThreads, "c", 10, "Maximum Number of templates executed in parallel")

View File

@ -1,7 +1,6 @@
package runner
import (
"bufio"
"context"
"fmt"
"net/http/cookiejar"
@ -50,6 +49,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat
ColoredOutput: !r.options.NoColor,
Colorizer: r.colorizer,
Decolorizer: r.decolorizer,
RateLimiter: r.ratelimiter,
})
case *requests.BulkHTTPRequest:
httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{
@ -73,6 +73,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat
StopAtFirstMatch: r.options.StopAtFirstMatch,
PF: r.pf,
Dialer: r.dialer,
RateLimiter: r.ratelimiter,
})
}
@ -87,9 +88,8 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat
wg := sizedwaitgroup.New(r.options.BulkSize)
scanner := bufio.NewScanner(strings.NewReader(r.input))
for scanner.Scan() {
URL := scanner.Text()
r.hm.Scan(func(k, _ []byte) error {
URL := string(k)
wg.Add()
go func(URL string) {
defer wg.Done()
@ -110,7 +110,9 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat
gologger.Warningf("[%s] Could not execute step: %s\n", r.colorizer.Colorizer.BrightBlue(template.ID), result.Error)
}
}(URL)
}
return nil
})
wg.Wait()
@ -132,9 +134,8 @@ func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflo
wg := sizedwaitgroup.New(r.options.BulkSize)
scanner := bufio.NewScanner(strings.NewReader(r.input))
for scanner.Scan() {
targetURL := scanner.Text()
r.hm.Scan(func(k, _ []byte) error {
targetURL := string(k)
wg.Add()
go func(targetURL string) {
@ -168,7 +169,8 @@ func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflo
}
}
}(targetURL)
}
return nil
})
wg.Wait()

View File

@ -2,9 +2,6 @@ package runner
import (
"bufio"
"fmt"
"io"
"io/ioutil"
"os"
"regexp"
"strings"
@ -13,24 +10,23 @@ import (
"github.com/pkg/errors"
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/hmap/store/hybrid"
"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
"github.com/projectdiscovery/nuclei/v2/pkg/collaborator"
"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
"github.com/projectdiscovery/nuclei/v2/pkg/globalratelimiter"
"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
"github.com/remeh/sizedwaitgroup"
"go.uber.org/ratelimit"
)
// Runner is a client for running the enumeration process.
type Runner struct {
input string
inputCount int64
tempFile string
traceLog tracelog.Log
@ -50,7 +46,11 @@ type Runner struct {
colorizer colorizer.NucleiColorizer
decolorizer *regexp.Regexp
// http dialer
// rate limiter
ratelimiter ratelimit.Limiter
// input deduplication
hm *hybrid.HybridMap
dialer *fastdialer.Dialer
}
@ -94,79 +94,67 @@ func New(options *Options) (*Runner, error) {
runner.readNucleiIgnoreFile()
}
// If we have stdin, write it to a new file
if options.Stdin {
tempInput, err := ioutil.TempFile("", "stdin-input-*")
if err != nil {
return nil, err
}
if _, err := io.Copy(tempInput, os.Stdin); err != nil {
return nil, err
}
runner.tempFile = tempInput.Name()
tempInput.Close()
}
// If we have single target, write it to a new file
if options.Target != "" {
tempInput, err := ioutil.TempFile("", "stdin-input-*")
if err != nil {
return nil, err
}
fmt.Fprintf(tempInput, "%s\n", options.Target)
runner.tempFile = tempInput.Name()
tempInput.Close()
if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil {
gologger.Fatalf("Could not create temporary input file: %s\n", err)
} else {
runner.hm = hm
}
// Setup input, handle a list of hosts as argument
var err error
var input *os.File
if options.Targets != "" {
input, err = os.Open(options.Targets)
} else if options.Stdin || options.Target != "" {
input, err = os.Open(runner.tempFile)
}
if err != nil {
gologger.Fatalf("Could not open targets file '%s': %s\n", options.Targets, err)
}
// Sanitize input and pre-compute total number of targets
var usedInput = make(map[string]struct{})
dupeCount := 0
sb := strings.Builder{}
scanner := bufio.NewScanner(input)
runner.inputCount = 0
dupeCount := 0
// Handle single target
if options.Target != "" {
runner.inputCount++
runner.hm.Set(options.Target, nil)
}
// Handle stdin
if options.Stdin {
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
url := strings.TrimSpace(scanner.Text())
// skip empty lines
if url == "" {
continue
}
// skip dupes
if _, ok := runner.hm.Get(url); ok {
dupeCount++
continue
}
for scanner.Scan() {
url := scanner.Text()
// skip empty lines
if url == "" {
continue
}
// deduplication
if _, ok := usedInput[url]; !ok {
usedInput[url] = struct{}{}
runner.inputCount++
// allocate global rate limiters
globalratelimiter.Add(url, options.RateLimit)
sb.WriteString(url)
sb.WriteString("\n")
} else {
dupeCount++
runner.hm.Set(url, nil)
}
}
input.Close()
runner.input = sb.String()
// Handle taget file
if options.Targets != "" {
input, err := os.Open(options.Targets)
if err != nil {
gologger.Fatalf("Could not open targets file '%s': %s\n", options.Targets, err)
}
scanner := bufio.NewScanner(input)
for scanner.Scan() {
url := strings.TrimSpace(scanner.Text())
// skip empty lines
if url == "" {
continue
}
// skip dupes
if _, ok := runner.hm.Get(url); ok {
dupeCount++
continue
}
runner.inputCount++
runner.hm.Set(url, nil)
}
input.Close()
}
if dupeCount > 0 {
gologger.Labelf("Supplied input was automatically deduplicated (%d removed).", dupeCount)
@ -199,11 +187,18 @@ func New(options *Options) (*Runner, error) {
}
// Create Dialer
var err error
runner.dialer, err = fastdialer.NewDialer(fastdialer.DefaultOptions)
if err != nil {
return nil, err
}
if options.RateLimit > 0 {
runner.ratelimiter = ratelimit.New(options.RateLimit)
} else {
runner.ratelimiter = ratelimit.NewUnlimited()
}
return runner, nil
}
@ -212,7 +207,7 @@ func (r *Runner) Close() {
if r.output != nil {
r.output.Close()
}
os.Remove(r.tempFile)
r.hm.Close()
if r.pf != nil {
r.pf.Close()
}

View File

@ -15,6 +15,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/requests"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
retryabledns "github.com/projectdiscovery/retryabledns"
"go.uber.org/ratelimit"
)
// DNSExecuter is a client for performing a DNS request
@ -32,6 +33,7 @@ type DNSExecuter struct {
template *templates.Template
dnsRequest *requests.DNSRequest
writer *bufwriter.Writer
ratelimiter ratelimit.Limiter
colorizer colorizer.NucleiColorizer
decolorizer *regexp.Regexp
@ -59,6 +61,7 @@ type DNSOptions struct {
Colorizer colorizer.NucleiColorizer
Decolorizer *regexp.Regexp
RateLimiter ratelimit.Limiter
}
// NewDNSExecuter creates a new DNS executer from a template
@ -79,6 +82,7 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
coloredOutput: options.ColoredOutput,
colorizer: options.Colorizer,
decolorizer: options.Decolorizer,
ratelimiter: options.RateLimiter,
}
return executer

View File

@ -26,7 +26,6 @@ import (
"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
"github.com/projectdiscovery/nuclei/v2/pkg/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/globalratelimiter"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
projetctfile "github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v2/pkg/requests"
@ -34,6 +33,7 @@ import (
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/remeh/sizedwaitgroup"
"go.uber.org/ratelimit"
"golang.org/x/net/proxy"
)
@ -65,6 +65,7 @@ type HTTPExecuter struct {
jsonRequest bool
noMeta bool
stopAtFirstMatch bool
ratelimiter ratelimit.Limiter
}
// HTTPOptions contains configuration options for the HTTP executer.
@ -89,6 +90,7 @@ type HTTPOptions struct {
ColoredOutput bool
StopAtFirstMatch bool
PF *projetctfile.ProjectFile
RateLimiter ratelimit.Limiter
Dialer *fastdialer.Dialer
}
@ -144,6 +146,7 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
decolorizer: options.Decolorizer,
stopAtFirstMatch: options.StopAtFirstMatch,
pf: options.PF,
ratelimiter: options.RateLimiter,
}
return executer, nil
@ -220,7 +223,7 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string)
go func(httpRequest *requests.HTTPRequest) {
defer swg.Done()
globalratelimiter.Take(reqURL)
e.ratelimiter.Take()
// If the request was built correctly then execute it
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
@ -351,7 +354,7 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result
result.Error = err
p.Drop(remaining)
} else {
globalratelimiter.Take(reqURL)
e.ratelimiter.Take()
// If the request was built correctly then execute it
format := "%s_" + strconv.Itoa(requestNumber)
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, format)

View File

@ -1,82 +0,0 @@
package globalratelimiter
import (
"sync"
"go.uber.org/ratelimit"
)
var defaultrwmutex sync.RWMutex
var defaultGlobalRateLimiter GlobalRateLimiter = GlobalRateLimiter{ratesLimiters: make(map[string]ratelimit.Limiter)}
type GlobalRateLimiter struct {
sync.RWMutex
ratesLimiters map[string]ratelimit.Limiter
}
func Add(k string, rateLimit int) {
defaultrwmutex.Lock()
defer defaultrwmutex.Unlock()
if rateLimit > 0 {
defaultGlobalRateLimiter.ratesLimiters[k] = ratelimit.New(rateLimit)
} else {
defaultGlobalRateLimiter.ratesLimiters[k] = ratelimit.NewUnlimited()
}
}
func Take(k string) {
rl := take(k)
rl.Take()
}
func take(k string) ratelimit.Limiter {
defaultrwmutex.RLock()
defer defaultrwmutex.RUnlock() //nolint
return defaultGlobalRateLimiter.ratesLimiters[k]
}
func Del(k string, rateLimit int) {
defaultrwmutex.Lock()
defer defaultrwmutex.Unlock() //nolint
delete(defaultGlobalRateLimiter.ratesLimiters, k)
}
func New() *GlobalRateLimiter {
var globalRateLimiter GlobalRateLimiter
globalRateLimiter.ratesLimiters = make(map[string]ratelimit.Limiter)
return &globalRateLimiter
}
func (grl *GlobalRateLimiter) Add(k string, rateLimit int) {
grl.Lock()
defer grl.Unlock()
if rateLimit > 0 {
grl.ratesLimiters[k] = ratelimit.New(rateLimit)
} else {
grl.ratesLimiters[k] = ratelimit.NewUnlimited()
}
}
func (grl *GlobalRateLimiter) take(k string) ratelimit.Limiter {
grl.RLock()
defer grl.RUnlock() //nolint
return grl.ratesLimiters[k]
}
func (grl *GlobalRateLimiter) Take(k string) {
rl := grl.take(k)
rl.Take()
}
func (grl *GlobalRateLimiter) Del(k string, rateLimit int) {
grl.Lock()
defer grl.Unlock()
delete(grl.ratesLimiters, k)
}