From eb080223d87c612d5dc6c0827597bf441e0699d2 Mon Sep 17 00:00:00 2001 From: Souvik Hazra Date: Sat, 31 Jul 2021 15:46:21 +0530 Subject: [PATCH 1/3] Added support for rate limit per minute --- v2/cmd/nuclei/main.go | 1 + v2/go.mod | 2 +- v2/go.sum | 4 ++++ v2/internal/runner/runner.go | 4 +++- v2/pkg/types/types.go | 2 ++ 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 6d381126..a3268c84 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -109,6 +109,7 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "rate-limit", "Rate-Limit", flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"), + flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"), flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"), flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "maximum number of templates to be executed in parallel"), ) diff --git a/v2/go.mod b/v2/go.mod index 4920a1f8..2e1a97ac 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -52,7 +52,7 @@ require ( github.com/xanzy/go-gitlab v0.44.0 go.uber.org/atomic v1.7.0 go.uber.org/multierr v1.6.0 - go.uber.org/ratelimit v0.1.0 + go.uber.org/ratelimit v0.2.0 golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect golang.org/x/net v0.0.0-20210521195947-fe42d452be8f golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 diff --git a/v2/go.sum b/v2/go.sum index 97fb14f0..f8512f4c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -40,6 +40,8 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE= github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= @@ -366,6 +368,8 @@ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw= go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y= +go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= +go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 017d3d86..5388f343 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -236,7 +236,9 @@ func New(options *types.Options) (*Runner, error) { } } - if options.RateLimit > 0 { + if options.RateLimitMinute > 0 { + runner.ratelimiter = ratelimit.New(options.RateLimitMinute, ratelimit.Per(60*time.Second)) + } else if options.RateLimit > 0 { runner.ratelimiter = ratelimit.New(options.RateLimit) } else { runner.ratelimiter = ratelimit.NewUnlimited() diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 291ec872..872b107e 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -70,6 +70,8 @@ type Options struct { Retries int // Rate-Limit is the maximum number of requests per specified target RateLimit int + // Rate-Limit is the maximum number of requests per minute for specified target + RateLimitMinute int // PageTimeout is the maximum time to wait for a page in seconds PageTimeout int // InteractionsCacheSize is the number of interaction-url->req to keep in cache at a time. From 25235523dafc5725d55d1a9ddaf6809353d86026 Mon Sep 17 00:00:00 2001 From: Souvik Hazra Date: Sun, 1 Aug 2021 11:00:38 +0530 Subject: [PATCH 2/3] Support for environment variables with .env file --- v2/cmd/nuclei/main.go | 4 ++++ v2/go.mod | 1 + v2/go.sum | 2 ++ v2/internal/runner/options.go | 8 ++++++++ v2/pkg/protocols/http/build_request.go | 14 ++++++++++++++ v2/pkg/protocols/http/http.go | 2 ++ v2/pkg/types/types.go | 2 ++ 7 files changed, 33 insertions(+) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index a3268c84..ebcc37eb 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -162,6 +162,10 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVar(&options.MetricsPort, "metrics-port", 9092, "port to expose nuclei metrics on"), ) + createGroup(flagSet, "env", "DotEnv", + flagSet.StringVar(&options.DotEnvPath, "env", ".env", "path of .env file to use"), + ) + _ = flagSet.Parse() if cfgFile != "" { diff --git a/v2/go.mod b/v2/go.mod index 2e1a97ac..8cb745c9 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -17,6 +17,7 @@ require ( github.com/gosuri/uiprogress v0.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect + github.com/joho/godotenv v1.3.0 // indirect github.com/json-iterator/go v1.1.10 github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible diff --git a/v2/go.sum b/v2/go.sum index f8512f4c..a85d89b2 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -183,6 +183,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index f60a3b6a..16284a3f 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -7,6 +7,7 @@ import ( "os" "strings" + "github.com/joho/godotenv" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" @@ -59,6 +60,13 @@ func ParseOptions(options *types.Options) { if err != nil { gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) } + + if len(options.DotEnvPath) > 0 { + err := godotenv.Load(options.DotEnvPath) + if err != nil { + gologger.Warning().Msgf("Failed loading .env file: %s\n", err) + } + } } // hasStdin returns true if we have stdin input diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 805f5e0b..78c3248b 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "net/url" + "os" "regexp" "strings" "time" @@ -129,6 +130,19 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest // request values. finalValues := generators.MergeMaps(generatorValues, values) + // read env variables + envVars := strings.Split(strings.Trim(r.request.EnvVars, " "), ",") + if len(envVars) > 0 { + envVarValues := make(map[string]interface{}) + for _, envVar := range envVars { + envVarValue := os.Getenv(envVar) + if len(envVarValue) > 0 { + envVarValues[envVar] = envVarValue + } + } + finalValues = generators.MergeMaps(finalValues, envVarValues) + } + // Evaulate the expressions for raw request if any. var err error rawRequest, err = expressions.Evaluate(rawRequest, finalValues) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index ecd297f8..2159aff4 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -73,6 +73,8 @@ type Request struct { // their history for being matched at the end. // Currently only works with sequential http requests. ReqCondition bool `yaml:"req-condition"` + // Dynamic ENV Variables + EnvVars string `yaml:"env-vars"` } // GetID returns the unique ID of the request if any. diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 872b107e..827c99e8 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -143,4 +143,6 @@ type Options struct { UpdateNuclei bool // NoUpdateTemplates disables checking for nuclei templates updates NoUpdateTemplates bool + // .env file path + DotEnvPath string } From 250b87aa01b804383e4b4f39e68fc9a9080918c3 Mon Sep 17 00:00:00 2001 From: Souvik Hazra Date: Mon, 2 Aug 2021 11:37:37 +0530 Subject: [PATCH 3/3] Revert "Support for environment variables with .env file" This reverts commit 25235523dafc5725d55d1a9ddaf6809353d86026. --- v2/cmd/nuclei/main.go | 4 ---- v2/go.mod | 1 - v2/go.sum | 2 -- v2/internal/runner/options.go | 8 -------- v2/pkg/protocols/http/build_request.go | 14 -------------- v2/pkg/protocols/http/http.go | 2 -- v2/pkg/types/types.go | 2 -- 7 files changed, 33 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index ebcc37eb..a3268c84 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -162,10 +162,6 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVar(&options.MetricsPort, "metrics-port", 9092, "port to expose nuclei metrics on"), ) - createGroup(flagSet, "env", "DotEnv", - flagSet.StringVar(&options.DotEnvPath, "env", ".env", "path of .env file to use"), - ) - _ = flagSet.Parse() if cfgFile != "" { diff --git a/v2/go.mod b/v2/go.mod index 8cb745c9..2e1a97ac 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -17,7 +17,6 @@ require ( github.com/gosuri/uiprogress v0.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect - github.com/joho/godotenv v1.3.0 // indirect github.com/json-iterator/go v1.1.10 github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible diff --git a/v2/go.sum b/v2/go.sum index a85d89b2..f8512f4c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -183,8 +183,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 16284a3f..f60a3b6a 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -7,7 +7,6 @@ import ( "os" "strings" - "github.com/joho/godotenv" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" @@ -60,13 +59,6 @@ func ParseOptions(options *types.Options) { if err != nil { gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) } - - if len(options.DotEnvPath) > 0 { - err := godotenv.Load(options.DotEnvPath) - if err != nil { - gologger.Warning().Msgf("Failed loading .env file: %s\n", err) - } - } } // hasStdin returns true if we have stdin input diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 78c3248b..805f5e0b 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -7,7 +7,6 @@ import ( "net" "net/http" "net/url" - "os" "regexp" "strings" "time" @@ -130,19 +129,6 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest // request values. finalValues := generators.MergeMaps(generatorValues, values) - // read env variables - envVars := strings.Split(strings.Trim(r.request.EnvVars, " "), ",") - if len(envVars) > 0 { - envVarValues := make(map[string]interface{}) - for _, envVar := range envVars { - envVarValue := os.Getenv(envVar) - if len(envVarValue) > 0 { - envVarValues[envVar] = envVarValue - } - } - finalValues = generators.MergeMaps(finalValues, envVarValues) - } - // Evaulate the expressions for raw request if any. var err error rawRequest, err = expressions.Evaluate(rawRequest, finalValues) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 2159aff4..ecd297f8 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -73,8 +73,6 @@ type Request struct { // their history for being matched at the end. // Currently only works with sequential http requests. ReqCondition bool `yaml:"req-condition"` - // Dynamic ENV Variables - EnvVars string `yaml:"env-vars"` } // GetID returns the unique ID of the request if any. diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 827c99e8..872b107e 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -143,6 +143,4 @@ type Options struct { UpdateNuclei bool // NoUpdateTemplates disables checking for nuclei templates updates NoUpdateTemplates bool - // .env file path - DotEnvPath string }