add ErrNoMoreRequests for generator (#3918)

* add ErrNoMoreRequests for generator

* fix gh repo name convention

* fix dirname in unit test
dev
Tarun Koyalwar 2023-07-13 00:51:06 +05:30 committed by GitHub
parent 4d1c9fe8b1
commit d51e058791
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 20 additions and 27 deletions

View File

@ -150,21 +150,9 @@ func (ctr *customTemplateGithubRepo) pullChanges(repoPath, githubToken string) e
return nil
}
// getLocalRepoClonePath returns the clone path.
// if same name repo directory exists from another owner then it appends the owner then and returns the path
// eg. for nuclei-templates directory exists for projectdiscovery owner, then for ehsandeep/nuclei-templates it will return nuclei-templates-ehsandeep
// All Custom github repos are cloned in the format of 'reponame-owner' for uniqueness
func (ctr *customTemplateGithubRepo) getLocalRepoClonePath(downloadPath string) string {
if fileutil.FolderExists(filepath.Join(downloadPath, ctr.reponame)) && !ctr.isRepoDirExists(filepath.Join(downloadPath, ctr.reponame)) {
return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner)
}
return filepath.Join(downloadPath, ctr.reponame)
}
// isRepoDirExists take the path and checks if the same repo or not
func (ctr *customTemplateGithubRepo) isRepoDirExists(repoPath string) bool {
r, _ := git.PlainOpen(repoPath)
local, _ := r.Config()
return local.User.Name == ctr.owner // repo already cloned no need to rename and clone
return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner)
}
// returns the auth object with username and github token as password

View File

@ -30,6 +30,6 @@ func TestDownloadCustomTemplatesFromGitHub(t *testing.T) {
ctm.Download(context.Background())
require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates"), "cloned directory does not exists")
require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-projectdiscovery"), "cloned directory does not exists")
require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-ehsandeep"), "cloned directory does not exists")
}

View File

@ -2,14 +2,13 @@ package fuzz
import (
"context"
"io"
"net/http"
"strings"
"github.com/corpix/uarand"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/retryablehttp-go"
sliceutil "github.com/projectdiscovery/utils/slice"
urlutil "github.com/projectdiscovery/utils/url"
@ -49,7 +48,6 @@ func (rule *Rule) executeQueryPartRule(input *ExecuteRuleInput, payload string)
requestURL.Params = temp
if qerr := rule.buildQueryInput(input, requestURL, input.InteractURLs); qerr != nil {
err = qerr
gologger.Error().Msgf("Could not build request for query part rule %v: %s\n", rule, err)
return false
}
cloned[i] = value // change back to previous value for temp
@ -89,7 +87,7 @@ func (rule *Rule) buildQueryInput(input *ExecuteRuleInput, parsed *urlutil.URL,
DynamicValues: input.Values,
}
if !input.Callback(request) {
return io.EOF
return types.ErrNoMoreRequests
}
return nil
}

View File

@ -2,11 +2,11 @@ package generators
import (
"bufio"
"io"
"path/filepath"
"strings"
"github.com/pkg/errors"
pkgTypes "github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/spf13/cast"
)
@ -60,7 +60,7 @@ func (generator *PayloadGenerator) loadPayloadsFromFile(filepath string) ([]stri
}
lines = append(lines, text)
}
if err := scanner.Err(); err != nil && !errors.Is(err, io.EOF) {
if err := scanner.Err(); err != nil && !errors.Is(err, pkgTypes.ErrNoMoreRequests) {
return lines, scanner.Err()
}
return lines, nil

View File

@ -1,7 +1,6 @@
package headless
import (
"io"
"net/url"
"strings"
"time"
@ -22,6 +21,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
urlutil "github.com/projectdiscovery/utils/url"
)
@ -206,7 +206,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads
Values: payloads,
BaseRequest: nil,
})
if err == io.EOF {
if err == types.ErrNoMoreRequests {
return nil
}
if err != nil {

View File

@ -65,7 +65,7 @@ func (r *requestGenerator) Total() int {
}
// Make creates a http request for the provided input.
// It returns io.EOF as error when all the requests have been exhausted.
// It returns ErrNoMoreRequests as error when all the requests have been exhausted.
func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, reqData string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
// value of `reqData` depends on the type of request specified in template
// 1. If request is raw request = reqData contains raw request (i.e http request dump)

View File

@ -134,7 +134,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
ctx := request.newContext(input)
generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
if err != nil {
if err == io.EOF {
if err == types.ErrNoMoreRequests {
break
}
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
@ -301,7 +301,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
Values: generated.dynamicValues,
BaseRequest: generated.request,
})
if err == io.EOF {
if err == types.ErrNoMoreRequests {
return nil
}
if err != nil {
@ -354,7 +354,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
defer cancel()
generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
if err != nil {
if err == io.EOF {
if err == types.ErrNoMoreRequests {
return true, nil
}
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))

View File

@ -13,6 +13,7 @@ var (
urlWithPortRegex = regexp.MustCompile(`^{{(BaseURL|RootURL)}}:(\d+)`)
// regex to detect traling slash in path (not applicable to raw requests)
trailingSlashregex = regexp.MustCompile(`^\Q{{\E[a-zA-Z]+\Q}}/\E`)
// ErrNoMoreRequests is internal error to
)
// HasTrailingSlash returns true if path(that has default variables) has trailing slash

View File

@ -1,6 +1,7 @@
package types
import (
"io"
"time"
"github.com/projectdiscovery/goflags"
@ -9,6 +10,11 @@ import (
fileutil "github.com/projectdiscovery/utils/file"
)
var (
// ErrNoMoreRequests is internal error to indicate that generator has no more requests to generate
ErrNoMoreRequests = io.EOF
)
// Options contains the configuration options for nuclei scanner.
type Options struct {
// Tags contains a list of tags to execute templates for. Multiple paths