add ErrNoMoreRequests for generator (#3918)

* add ErrNoMoreRequests for generator

* fix gh repo name convention

* fix dirname in unit test

v2/pkg/external/customtemplates/github.go

@@ -150,21 +150,9 @@ func (ctr *customTemplateGithubRepo) pullChanges(repoPath, githubToken string) e
 	return nil
 }
 
-// getLocalRepoClonePath returns the clone path.
-// if same name repo directory exists from another owner then it appends the owner then and returns the path
-// eg. for nuclei-templates directory exists for projectdiscovery owner, then for ehsandeep/nuclei-templates it will return nuclei-templates-ehsandeep
+// All Custom github repos are cloned in the format of 'reponame-owner' for uniqueness
 func (ctr *customTemplateGithubRepo) getLocalRepoClonePath(downloadPath string) string {
-	if fileutil.FolderExists(filepath.Join(downloadPath, ctr.reponame)) && !ctr.isRepoDirExists(filepath.Join(downloadPath, ctr.reponame)) {
-		return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner)
-	}
-	return filepath.Join(downloadPath, ctr.reponame)
-}
-
-// isRepoDirExists take the path and checks if the same repo or not
-func (ctr *customTemplateGithubRepo) isRepoDirExists(repoPath string) bool {
-	r, _ := git.PlainOpen(repoPath)
-	local, _ := r.Config()
-	return local.User.Name == ctr.owner // repo already cloned no need to rename and clone
+	return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner)
 }
 
 // returns the auth object with username and github token as password

v2/pkg/external/customtemplates/github_test.go

@@ -30,6 +30,6 @@ func TestDownloadCustomTemplatesFromGitHub(t *testing.T) {
 
 	ctm.Download(context.Background())
 
-	require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates"), "cloned directory does not exists")
+	require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-projectdiscovery"), "cloned directory does not exists")
 	require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-ehsandeep"), "cloned directory does not exists")
 }

v2/pkg/protocols/common/fuzz/parts.go

@@ -2,14 +2,13 @@ package fuzz
 
 import (
 	"context"
-	"io"
 	"net/http"
 	"strings"
 
 	"github.com/corpix/uarand"
-	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/retryablehttp-go"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	urlutil "github.com/projectdiscovery/utils/url"
@@ -49,7 +48,6 @@ func (rule *Rule) executeQueryPartRule(input *ExecuteRuleInput, payload string)
 				requestURL.Params = temp
 				if qerr := rule.buildQueryInput(input, requestURL, input.InteractURLs); qerr != nil {
 					err = qerr
-					gologger.Error().Msgf("Could not build request for query part rule %v: %s\n", rule, err)
 					return false
 				}
 				cloned[i] = value // change back to previous value for temp
@@ -89,7 +87,7 @@ func (rule *Rule) buildQueryInput(input *ExecuteRuleInput, parsed *urlutil.URL,
 		DynamicValues: input.Values,
 	}
 	if !input.Callback(request) {
-		return io.EOF
+		return types.ErrNoMoreRequests
 	}
 	return nil
 }

v2/pkg/protocols/common/generators/load.go

@@ -2,11 +2,11 @@ package generators
 
 import (
 	"bufio"
-	"io"
 	"path/filepath"
 	"strings"
 
 	"github.com/pkg/errors"
+	pkgTypes "github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/spf13/cast"
 )
 
@@ -60,7 +60,7 @@ func (generator *PayloadGenerator) loadPayloadsFromFile(filepath string) ([]stri
 		}
 		lines = append(lines, text)
 	}
-	if err := scanner.Err(); err != nil && !errors.Is(err, io.EOF) {
+	if err := scanner.Err(); err != nil && !errors.Is(err, pkgTypes.ErrNoMoreRequests) {
 		return lines, scanner.Err()
 	}
 	return lines, nil

v2/pkg/protocols/headless/request.go

@@ -1,7 +1,6 @@
 package headless
 
 import (
-	"io"
 	"net/url"
 	"strings"
 	"time"
@@ -22,6 +21,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
 	protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
 	templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	urlutil "github.com/projectdiscovery/utils/url"
 )
 
@@ -206,7 +206,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads
 			Values:      payloads,
 			BaseRequest: nil,
 		})
-		if err == io.EOF {
+		if err == types.ErrNoMoreRequests {
 			return nil
 		}
 		if err != nil {

v2/pkg/protocols/http/build_request.go

@@ -65,7 +65,7 @@ func (r *requestGenerator) Total() int {
 }
 
 // Make creates a http request for the provided input.
-// It returns io.EOF as error when all the requests have been exhausted.
+// It returns ErrNoMoreRequests as error when all the requests have been exhausted.
 func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, reqData string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
 	// value of `reqData` depends on the type of request specified in template
 	// 1. If request is raw request =  reqData contains raw request (i.e http request dump)

v2/pkg/protocols/http/request.go

@@ -134,7 +134,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		ctx := request.newContext(input)
 		generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
 		if err != nil {
-			if err == io.EOF {
+			if err == types.ErrNoMoreRequests {
 				break
 			}
 			request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
@@ -301,7 +301,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
 				Values:      generated.dynamicValues,
 				BaseRequest: generated.request,
 			})
-			if err == io.EOF {
+			if err == types.ErrNoMoreRequests {
 				return nil
 			}
 			if err != nil {
@@ -354,7 +354,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 			defer cancel()
 			generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
 			if err != nil {
-				if err == io.EOF {
+				if err == types.ErrNoMoreRequests {
 					return true, nil
 				}
 				request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))

v2/pkg/protocols/http/utils/requtils.go

@@ -13,6 +13,7 @@ var (
 	urlWithPortRegex = regexp.MustCompile(`^{{(BaseURL|RootURL)}}:(\d+)`)
 	// regex to detect traling slash in path (not applicable to raw requests)
 	trailingSlashregex = regexp.MustCompile(`^\Q{{\E[a-zA-Z]+\Q}}/\E`)
+	// ErrNoMoreRequests is internal error to
 )
 
 // HasTrailingSlash returns true if path(that has default variables) has trailing slash

v2/pkg/types/types.go

@@ -1,6 +1,7 @@
 package types
 
 import (
+	"io"
 	"time"
 
 	"github.com/projectdiscovery/goflags"
@@ -9,6 +10,11 @@ import (
 	fileutil "github.com/projectdiscovery/utils/file"
 )
 
+var (
+	// ErrNoMoreRequests is internal error to indicate that generator has no more requests to generate
+	ErrNoMoreRequests = io.EOF
+)
+
 // Options contains the configuration options for nuclei scanner.
 type Options struct {
 	// Tags contains a list of tags to execute templates for. Multiple paths