diff --git a/v2/pkg/external/customtemplates/github.go b/v2/pkg/external/customtemplates/github.go index e0cd75d9..d9317ca3 100644 --- a/v2/pkg/external/customtemplates/github.go +++ b/v2/pkg/external/customtemplates/github.go @@ -150,21 +150,9 @@ func (ctr *customTemplateGithubRepo) pullChanges(repoPath, githubToken string) e return nil } -// getLocalRepoClonePath returns the clone path. -// if same name repo directory exists from another owner then it appends the owner then and returns the path -// eg. for nuclei-templates directory exists for projectdiscovery owner, then for ehsandeep/nuclei-templates it will return nuclei-templates-ehsandeep +// All Custom github repos are cloned in the format of 'reponame-owner' for uniqueness func (ctr *customTemplateGithubRepo) getLocalRepoClonePath(downloadPath string) string { - if fileutil.FolderExists(filepath.Join(downloadPath, ctr.reponame)) && !ctr.isRepoDirExists(filepath.Join(downloadPath, ctr.reponame)) { - return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner) - } - return filepath.Join(downloadPath, ctr.reponame) -} - -// isRepoDirExists take the path and checks if the same repo or not -func (ctr *customTemplateGithubRepo) isRepoDirExists(repoPath string) bool { - r, _ := git.PlainOpen(repoPath) - local, _ := r.Config() - return local.User.Name == ctr.owner // repo already cloned no need to rename and clone + return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner) } // returns the auth object with username and github token as password diff --git a/v2/pkg/external/customtemplates/github_test.go b/v2/pkg/external/customtemplates/github_test.go index 60d46956..e21a62a9 100644 --- a/v2/pkg/external/customtemplates/github_test.go +++ b/v2/pkg/external/customtemplates/github_test.go @@ -30,6 +30,6 @@ func TestDownloadCustomTemplatesFromGitHub(t *testing.T) { ctm.Download(context.Background()) - require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates"), "cloned directory does not exists") + require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-projectdiscovery"), "cloned directory does not exists") require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-ehsandeep"), "cloned directory does not exists") } diff --git a/v2/pkg/protocols/common/fuzz/parts.go b/v2/pkg/protocols/common/fuzz/parts.go index 43e0f420..0e09aaf8 100644 --- a/v2/pkg/protocols/common/fuzz/parts.go +++ b/v2/pkg/protocols/common/fuzz/parts.go @@ -2,14 +2,13 @@ package fuzz import ( "context" - "io" "net/http" "strings" "github.com/corpix/uarand" - "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/retryablehttp-go" sliceutil "github.com/projectdiscovery/utils/slice" urlutil "github.com/projectdiscovery/utils/url" @@ -49,7 +48,6 @@ func (rule *Rule) executeQueryPartRule(input *ExecuteRuleInput, payload string) requestURL.Params = temp if qerr := rule.buildQueryInput(input, requestURL, input.InteractURLs); qerr != nil { err = qerr - gologger.Error().Msgf("Could not build request for query part rule %v: %s\n", rule, err) return false } cloned[i] = value // change back to previous value for temp @@ -89,7 +87,7 @@ func (rule *Rule) buildQueryInput(input *ExecuteRuleInput, parsed *urlutil.URL, DynamicValues: input.Values, } if !input.Callback(request) { - return io.EOF + return types.ErrNoMoreRequests } return nil } diff --git a/v2/pkg/protocols/common/generators/load.go b/v2/pkg/protocols/common/generators/load.go index de419488..612f4e33 100644 --- a/v2/pkg/protocols/common/generators/load.go +++ b/v2/pkg/protocols/common/generators/load.go @@ -2,11 +2,11 @@ package generators import ( "bufio" - "io" "path/filepath" "strings" "github.com/pkg/errors" + pkgTypes "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/spf13/cast" ) @@ -60,7 +60,7 @@ func (generator *PayloadGenerator) loadPayloadsFromFile(filepath string) ([]stri } lines = append(lines, text) } - if err := scanner.Err(); err != nil && !errors.Is(err, io.EOF) { + if err := scanner.Err(); err != nil && !errors.Is(err, pkgTypes.ErrNoMoreRequests) { return lines, scanner.Err() } return lines, nil diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 9365d142..b7f40eb4 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -1,7 +1,6 @@ package headless import ( - "io" "net/url" "strings" "time" @@ -22,6 +21,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + "github.com/projectdiscovery/nuclei/v2/pkg/types" urlutil "github.com/projectdiscovery/utils/url" ) @@ -206,7 +206,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads Values: payloads, BaseRequest: nil, }) - if err == io.EOF { + if err == types.ErrNoMoreRequests { return nil } if err != nil { diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 1b768402..eecd9ead 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -65,7 +65,7 @@ func (r *requestGenerator) Total() int { } // Make creates a http request for the provided input. -// It returns io.EOF as error when all the requests have been exhausted. +// It returns ErrNoMoreRequests as error when all the requests have been exhausted. func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, reqData string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) { // value of `reqData` depends on the type of request specified in template // 1. If request is raw request = reqData contains raw request (i.e http request dump) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 127b7a0e..04469edc 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -134,7 +134,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV ctx := request.newContext(input) generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues) if err != nil { - if err == io.EOF { + if err == types.ErrNoMoreRequests { break } request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) @@ -301,7 +301,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous Values: generated.dynamicValues, BaseRequest: generated.request, }) - if err == io.EOF { + if err == types.ErrNoMoreRequests { return nil } if err != nil { @@ -354,7 +354,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa defer cancel() generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue) if err != nil { - if err == io.EOF { + if err == types.ErrNoMoreRequests { return true, nil } request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) diff --git a/v2/pkg/protocols/http/utils/requtils.go b/v2/pkg/protocols/http/utils/requtils.go index cfb75c15..7be1f225 100644 --- a/v2/pkg/protocols/http/utils/requtils.go +++ b/v2/pkg/protocols/http/utils/requtils.go @@ -13,6 +13,7 @@ var ( urlWithPortRegex = regexp.MustCompile(`^{{(BaseURL|RootURL)}}:(\d+)`) // regex to detect traling slash in path (not applicable to raw requests) trailingSlashregex = regexp.MustCompile(`^\Q{{\E[a-zA-Z]+\Q}}/\E`) + // ErrNoMoreRequests is internal error to ) // HasTrailingSlash returns true if path(that has default variables) has trailing slash diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index b3880c05..ada0aed7 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -1,6 +1,7 @@ package types import ( + "io" "time" "github.com/projectdiscovery/goflags" @@ -9,6 +10,11 @@ import ( fileutil "github.com/projectdiscovery/utils/file" ) +var ( + // ErrNoMoreRequests is internal error to indicate that generator has no more requests to generate + ErrNoMoreRequests = io.EOF +) + // Options contains the configuration options for nuclei scanner. type Options struct { // Tags contains a list of tags to execute templates for. Multiple paths