Merge branch 'dev' into dependabot/go_modules/v2/dev/github.com/aws/aws-sdk-go-1.44.8

v2/cmd/nuclei/main.go

@@ -235,6 +235,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	}
 	cleanupOldResumeFiles()
 }
+
 func cleanupOldResumeFiles() {
 	root, err := config.GetConfigDir()
 	if err != nil {
@@ -246,6 +247,7 @@ func cleanupOldResumeFiles() {
 	}
 	_ = fileutil.DeleteFilesOlderThan(root, filter)
 }
+
 func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) {
 	flagSet.SetGroup(groupName, description)
 	for _, currentFlag := range flags {

v2/go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/karlseguin/ccache v2.0.3+incompatible
-	github.com/karrick/godirwalk v1.17.0
+	github.com/karrick/godirwalk v1.17.0 // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/miekg/dns v1.1.48
 	github.com/olekukonko/tablewriter v0.0.5
@@ -33,11 +33,11 @@ require (
 	github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa
-	github.com/projectdiscovery/interactsh v1.0.3
+	github.com/projectdiscovery/interactsh v1.0.4
 	github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df
 	github.com/projectdiscovery/rawhttp v0.0.8-0.20220504112210-ae777c1ccd6b
 	github.com/projectdiscovery/retryabledns v1.0.13
-	github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220414143248-bb6eabffa43e
+	github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d
 	github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833
 	github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6
 	github.com/remeh/sizedwaitgroup v1.0.0

v2/go.sum

@@ -461,6 +461,8 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:
 github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE=
 github.com/projectdiscovery/interactsh v1.0.3 h1:QFL/fFTfvhLeIRFBSguWP8r6h8vf65Bhkf8wzKU2/qQ=
 github.com/projectdiscovery/interactsh v1.0.3/go.mod h1:3943djLJ4SZBZfJ7A2XEy8xspfFmT0BDiN9FCW7PAXY=
+github.com/projectdiscovery/interactsh v1.0.4 h1:73HHOeqGduU1lkIqeYkuEfzdejXRqxQJFZCEbQ1uvkU=
+github.com/projectdiscovery/interactsh v1.0.4/go.mod h1:57Xl3Q59aTb8VaFNGkCPO8pWTyTYhlM33p+HDhL8Ygw=
 github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE=
 github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
 github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
@@ -490,6 +492,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1
 github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
 github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220414143248-bb6eabffa43e h1:ohoSKR8w4GzGjmOaqdCa8pvHm3qbAyv489skpyrkCX0=
 github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220414143248-bb6eabffa43e/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE=
+github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d h1:VR+tDkedzHIp1pGKIDcfPFt7J8KjcjxGsJvBAP6RXFQ=
+github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE=
 github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d h1:wIQPYRZEwTeJuoZLv3NT9r+il2fAv1ObRzTdHkNgOxk=
 github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4=
 github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=

v2/internal/runner/templates.go

@@ -1,11 +1,11 @@
 package runner
 
 import (
+	"io/fs"
 	"os"
+	"path/filepath"
 	"strings"
 
-	"github.com/karrick/godirwalk"
-
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
@@ -40,9 +40,13 @@ func (r *Runner) listAvailableTemplates() {
 		r.templatesConfig.TemplateVersion,
 		r.templatesConfig.TemplatesDirectory,
 	)
-	err := directoryWalker(
+	err := filepath.WalkDir(
 		r.templatesConfig.TemplatesDirectory,
-		func(path string, d *godirwalk.Dirent) error {
+		func(path string, d fs.DirEntry, err error) error {
+			// continue on errors
+			if err != nil {
+				return nil
+			}
 			if d.IsDir() && path != r.templatesConfig.TemplatesDirectory {
 				gologger.Print().Msgf("\n%s:\n\n", r.colorizer.Bold(r.colorizer.BgBrightBlue(d.Name())).String())
 			} else if strings.HasSuffix(path, ".yaml") {
@@ -56,13 +60,3 @@ func (r *Runner) listAvailableTemplates() {
 		gologger.Error().Msgf("Could not find templates in directory '%s': %s\n", r.templatesConfig.TemplatesDirectory, err)
 	}
 }
-
-func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error {
-	return godirwalk.Walk(fsPath, &godirwalk.Options{
-		Callback: callback,
-		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
-			return godirwalk.SkipNode
-		},
-		Unsorted: true,
-	})
-}

v2/pkg/catalog/config/config.go

@@ -27,7 +27,7 @@ type Config struct {
 const nucleiConfigFilename = ".templates-config.json"
 
 // Version is the current version of nuclei
-const Version = `2.6.9`
+const Version = `2.7.0`
 
 func getConfigDetails() (string, error) {
 	configDir, err := GetConfigDir()

v2/pkg/catalog/find.go

@@ -1,11 +1,11 @@
 package catalog
 
 import (
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/karrick/godirwalk"
 	"github.com/pkg/errors"
 
 	"github.com/projectdiscovery/gologger"
@@ -136,12 +136,13 @@ func (c *Catalog) findFileMatches(absPath string, processed map[string]struct{})
 // findDirectoryMatches finds matches for templates from a directory
 func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) {
 	var results []string
-	err := godirwalk.Walk(absPath, &godirwalk.Options{
+	err := filepath.WalkDir(
-		Unsorted: true,
+		absPath,
-		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
+		func(path string, d fs.DirEntry, err error) error {
-			return godirwalk.SkipNode
+			// continue on errors
-		},
+			if err != nil {
-		Callback: func(path string, d *godirwalk.Dirent) error {
+				return nil
+			}
 			if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
 				if _, ok := processed[path]; !ok {
 					results = append(results, path)
@@ -150,6 +151,6 @@ func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]stru
 			}
 			return nil
 		},
-	})
+	)
 	return results, err
 }

v2/pkg/protocols/common/compare/compare.go

@@ -1,6 +1,8 @@
 package compare
 
-import "strings"
+import (
+	"strings"
+)
 
 // StringSlice compares two string slices for equality
 func StringSlice(a, b []string) bool {

v2/pkg/protocols/file/find.go

@@ -2,11 +2,11 @@ package file
 
 import (
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/karrick/godirwalk"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/fileutil"
 	"github.com/projectdiscovery/folderutil"
@@ -86,12 +86,13 @@ func (request *Request) findFileMatches(absPath string, processed map[string]str
 
 // findDirectoryMatches finds matches for templates from a directory
 func (request *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error {
-	err := godirwalk.Walk(absPath, &godirwalk.Options{
+	err := filepath.WalkDir(
-		Unsorted: true,
+		absPath,
-		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
+		func(path string, d fs.DirEntry, err error) error {
-			return godirwalk.SkipNode
+			// continue on errors
-		},
+			if err != nil {
-		Callback: func(path string, d *godirwalk.Dirent) error {
+				return nil
+			}
 			if d.IsDir() {
 				return nil
 			}
@@ -104,7 +105,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin
 			}
 			return nil
 		},
-	})
+	)
 	return err
 }
 

v2/pkg/protocols/offlinehttp/find.go

@@ -1,11 +1,11 @@
 package offlinehttp
 
 import (
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/karrick/godirwalk"
 	"github.com/pkg/errors"
 )
 
@@ -80,12 +80,13 @@ func (request *Request) findFileMatches(absPath string, processed map[string]str
 
 // findDirectoryMatches finds matches for templates from a directory
 func (request *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error {
-	err := godirwalk.Walk(absPath, &godirwalk.Options{
+	err := filepath.WalkDir(
-		Unsorted: true,
+		absPath,
-		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
+		func(p string, d fs.DirEntry, err error) error {
-			return godirwalk.SkipNode
+			// continue on errors
-		},
+			if err != nil {
-		Callback: func(p string, d *godirwalk.Dirent) error {
+				return nil
+			}
 			if d.IsDir() {
 				return nil
 			}
@@ -98,6 +99,6 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin
 			}
 			return nil
 		},
-	})
+	)
 	return err
 }