From a534b9f06c6bdf36a50bffb9d1134edbc208c291 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 8 May 2022 08:52:21 +0200 Subject: [PATCH 1/2] Fixing directory walk error check on windows (#1951) * Fixing directory walk error check on windows * moving check to helper package * replacing godirwalk with standard library --- v2/cmd/nuclei/main.go | 2 ++ v2/go.mod | 2 +- v2/internal/runner/templates.go | 22 ++++++++-------------- v2/pkg/catalog/find.go | 17 +++++++++-------- v2/pkg/protocols/common/compare/compare.go | 4 +++- v2/pkg/protocols/file/find.go | 17 +++++++++-------- v2/pkg/protocols/offlinehttp/find.go | 17 +++++++++-------- 7 files changed, 41 insertions(+), 40 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 877fd635..a4672054 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -235,6 +235,7 @@ on extensive configurability, massive extensibility and ease of use.`) } cleanupOldResumeFiles() } + func cleanupOldResumeFiles() { root, err := config.GetConfigDir() if err != nil { @@ -246,6 +247,7 @@ func cleanupOldResumeFiles() { } _ = fileutil.DeleteFilesOlderThan(root, filter) } + func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) { flagSet.SetGroup(groupName, description) for _, currentFlag := range flags { diff --git a/v2/go.mod b/v2/go.mod index 2bf8d12a..05dba0e9 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -19,7 +19,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible - github.com/karrick/godirwalk v1.17.0 + github.com/karrick/godirwalk v1.17.0 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible github.com/miekg/dns v1.1.48 github.com/olekukonko/tablewriter v0.0.5 diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index 23331b5f..fd6d6abc 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -1,11 +1,11 @@ package runner import ( + "io/fs" "os" + "path/filepath" "strings" - "github.com/karrick/godirwalk" - "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/parsers" "github.com/projectdiscovery/nuclei/v2/pkg/templates" @@ -40,9 +40,13 @@ func (r *Runner) listAvailableTemplates() { r.templatesConfig.TemplateVersion, r.templatesConfig.TemplatesDirectory, ) - err := directoryWalker( + err := filepath.WalkDir( r.templatesConfig.TemplatesDirectory, - func(path string, d *godirwalk.Dirent) error { + func(path string, d fs.DirEntry, err error) error { + // continue on errors + if err != nil { + return nil + } if d.IsDir() && path != r.templatesConfig.TemplatesDirectory { gologger.Print().Msgf("\n%s:\n\n", r.colorizer.Bold(r.colorizer.BgBrightBlue(d.Name())).String()) } else if strings.HasSuffix(path, ".yaml") { @@ -56,13 +60,3 @@ func (r *Runner) listAvailableTemplates() { gologger.Error().Msgf("Could not find templates in directory '%s': %s\n", r.templatesConfig.TemplatesDirectory, err) } } - -func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error { - return godirwalk.Walk(fsPath, &godirwalk.Options{ - Callback: callback, - ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction { - return godirwalk.SkipNode - }, - Unsorted: true, - }) -} diff --git a/v2/pkg/catalog/find.go b/v2/pkg/catalog/find.go index ed5c950d..fc3fc225 100644 --- a/v2/pkg/catalog/find.go +++ b/v2/pkg/catalog/find.go @@ -1,11 +1,11 @@ package catalog import ( + "io/fs" "os" "path/filepath" "strings" - "github.com/karrick/godirwalk" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" @@ -136,12 +136,13 @@ func (c *Catalog) findFileMatches(absPath string, processed map[string]struct{}) // findDirectoryMatches finds matches for templates from a directory func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) { var results []string - err := godirwalk.Walk(absPath, &godirwalk.Options{ - Unsorted: true, - ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction { - return godirwalk.SkipNode - }, - Callback: func(path string, d *godirwalk.Dirent) error { + err := filepath.WalkDir( + absPath, + func(path string, d fs.DirEntry, err error) error { + // continue on errors + if err != nil { + return nil + } if !d.IsDir() && strings.HasSuffix(path, ".yaml") { if _, ok := processed[path]; !ok { results = append(results, path) @@ -150,6 +151,6 @@ func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]stru } return nil }, - }) + ) return results, err } diff --git a/v2/pkg/protocols/common/compare/compare.go b/v2/pkg/protocols/common/compare/compare.go index 683b5fac..6e497045 100644 --- a/v2/pkg/protocols/common/compare/compare.go +++ b/v2/pkg/protocols/common/compare/compare.go @@ -1,6 +1,8 @@ package compare -import "strings" +import ( + "strings" +) // StringSlice compares two string slices for equality func StringSlice(a, b []string) bool { diff --git a/v2/pkg/protocols/file/find.go b/v2/pkg/protocols/file/find.go index 6ad22139..ad3607ed 100644 --- a/v2/pkg/protocols/file/find.go +++ b/v2/pkg/protocols/file/find.go @@ -2,11 +2,11 @@ package file import ( "io" + "io/fs" "os" "path/filepath" "strings" - "github.com/karrick/godirwalk" "github.com/pkg/errors" "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/folderutil" @@ -86,12 +86,13 @@ func (request *Request) findFileMatches(absPath string, processed map[string]str // findDirectoryMatches finds matches for templates from a directory func (request *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error { - err := godirwalk.Walk(absPath, &godirwalk.Options{ - Unsorted: true, - ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction { - return godirwalk.SkipNode - }, - Callback: func(path string, d *godirwalk.Dirent) error { + err := filepath.WalkDir( + absPath, + func(path string, d fs.DirEntry, err error) error { + // continue on errors + if err != nil { + return nil + } if d.IsDir() { return nil } @@ -104,7 +105,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin } return nil }, - }) + ) return err } diff --git a/v2/pkg/protocols/offlinehttp/find.go b/v2/pkg/protocols/offlinehttp/find.go index 2c876dab..ec6fe1d9 100644 --- a/v2/pkg/protocols/offlinehttp/find.go +++ b/v2/pkg/protocols/offlinehttp/find.go @@ -1,11 +1,11 @@ package offlinehttp import ( + "io/fs" "os" "path/filepath" "strings" - "github.com/karrick/godirwalk" "github.com/pkg/errors" ) @@ -80,12 +80,13 @@ func (request *Request) findFileMatches(absPath string, processed map[string]str // findDirectoryMatches finds matches for templates from a directory func (request *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error { - err := godirwalk.Walk(absPath, &godirwalk.Options{ - Unsorted: true, - ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction { - return godirwalk.SkipNode - }, - Callback: func(p string, d *godirwalk.Dirent) error { + err := filepath.WalkDir( + absPath, + func(p string, d fs.DirEntry, err error) error { + // continue on errors + if err != nil { + return nil + } if d.IsDir() { return nil } @@ -98,6 +99,6 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin } return nil }, - }) + ) return err } From 474d66c09dc9e076073472984e8a2763c384b9bf Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 8 May 2022 15:29:33 +0530 Subject: [PATCH 2/2] go mod + version update --- v2/go.mod | 4 ++-- v2/go.sum | 4 ++++ v2/pkg/catalog/config/config.go | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 05dba0e9..dfda9c52 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -33,11 +33,11 @@ require ( github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa - github.com/projectdiscovery/interactsh v1.0.3 + github.com/projectdiscovery/interactsh v1.0.4 github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df github.com/projectdiscovery/rawhttp v0.0.8-0.20220504112210-ae777c1ccd6b github.com/projectdiscovery/retryabledns v1.0.13 - github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220414143248-bb6eabffa43e + github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833 github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/v2/go.sum b/v2/go.sum index 591a791a..12c2b358 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -461,6 +461,8 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1: github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= github.com/projectdiscovery/interactsh v1.0.3 h1:QFL/fFTfvhLeIRFBSguWP8r6h8vf65Bhkf8wzKU2/qQ= github.com/projectdiscovery/interactsh v1.0.3/go.mod h1:3943djLJ4SZBZfJ7A2XEy8xspfFmT0BDiN9FCW7PAXY= +github.com/projectdiscovery/interactsh v1.0.4 h1:73HHOeqGduU1lkIqeYkuEfzdejXRqxQJFZCEbQ1uvkU= +github.com/projectdiscovery/interactsh v1.0.4/go.mod h1:57Xl3Q59aTb8VaFNGkCPO8pWTyTYhlM33p+HDhL8Ygw= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= @@ -490,6 +492,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1 github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220414143248-bb6eabffa43e h1:ohoSKR8w4GzGjmOaqdCa8pvHm3qbAyv489skpyrkCX0= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220414143248-bb6eabffa43e/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= +github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d h1:VR+tDkedzHIp1pGKIDcfPFt7J8KjcjxGsJvBAP6RXFQ= +github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d h1:wIQPYRZEwTeJuoZLv3NT9r+il2fAv1ObRzTdHkNgOxk= github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4= github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 1b2d7bbd..8450b841 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -27,7 +27,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.6.9` +const Version = `2.7.0` func getConfigDetails() (string, error) { configDir, err := GetConfigDir()