Extending deny list to support filenames and folders (#1260)

* Extending deny list to support filenames and folders

* fixing field name

* adding missing edge case with relative path + filename

* handling root path + relative path

* Improving matchers to handle all deny cases

v2/pkg/protocols/file/file.go

@@ -1,6 +1,7 @@
 package file
 
 import (
+	"path/filepath"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -19,13 +20,13 @@ type Request struct {
 	//   - value: '[]string{".txt", ".go", ".json"}'
 	Extensions []string `yaml:"extensions,omitempty" jsonschema:"title=extensions to match,description=List of extensions to perform matching on"`
 	// description: |
-	//   ExtensionDenylist is the list of file extensions to deny during matching.
+	//   DenyList is the list of file, directories or extensions to deny during matching.
 	//
 	//   By default, it contains some non-interesting extensions that are hardcoded
 	//   in nuclei.
 	// examples:
 	//   - value: '[]string{".avi", ".mov", ".mp3"}'
-	ExtensionDenylist []string `yaml:"denylist,omitempty" jsonschema:"title=extensions to deny match,description=List of file extensions to deny during matching"`
+	DenyList []string `yaml:"denylist,omitempty" jsonschema:"title=denylist, directories and extentions to deny match,description=List of files, directories and extensions to deny during matching"`
 
 	// ID is the optional id of the request
 	ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=ID is the optional ID for the request"`
@@ -41,9 +42,9 @@ type Request struct {
 	CompiledOperators *operators.Operators `yaml:"-"`
 
 	// cache any variables that may be needed for operation.
-	options           *protocols.ExecuterOptions
-	extensions        map[string]struct{}
-	extensionDenylist map[string]struct{}
+	options    *protocols.ExecuterOptions
+	extensions map[string]struct{}
+	denyList   map[string]struct{}
 
 	// description: |
 	//   NoRecursive specifies whether to not do recursive checks if folders are provided.
@@ -89,7 +90,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 	request.options = options
 
 	request.extensions = make(map[string]struct{})
-	request.extensionDenylist = make(map[string]struct{})
+	request.denyList = make(map[string]struct{})
 
 	for _, extension := range request.Extensions {
 		if extension == "all" {
@@ -101,17 +102,17 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 			request.extensions[extension] = struct{}{}
 		}
 	}
-	for _, extension := range defaultDenylist {
-		if !strings.HasPrefix(extension, ".") {
-			extension = "." + extension
+	// process default denylist (extensions)
+	for _, excludeItem := range defaultDenylist {
+		if !strings.HasPrefix(excludeItem, ".") {
+			excludeItem = "." + excludeItem
 		}
-		request.extensionDenylist[extension] = struct{}{}
+		request.denyList[excludeItem] = struct{}{}
 	}
-	for _, extension := range request.ExtensionDenylist {
-		if !strings.HasPrefix(extension, ".") {
-			extension = "." + extension
-		}
-		request.extensionDenylist[extension] = struct{}{}
+	for _, excludeItem := range request.DenyList {
+		request.denyList[excludeItem] = struct{}{}
+		// also add a cleaned version as the exclusion path can be dirty (eg. /a/b/c, /a/b/c/, a///b///c/../d)
+		request.denyList[filepath.Clean(excludeItem)] = struct{}{}
 	}
 	return nil
 }

v2/pkg/protocols/file/file_test.go

@@ -16,11 +16,11 @@ func TestFileCompile(t *testing.T) {
 	testutils.Init(options)
 	templateID := "testing-file"
 	request := &Request{
-		ID:                templateID,
-		MaxSize:           1024,
-		NoRecursive:       false,
-		Extensions:        []string{"all", ".lock"},
-		ExtensionDenylist: []string{".go"},
+		ID:          templateID,
+		MaxSize:     1024,
+		NoRecursive: false,
+		Extensions:  []string{"all", ".lock"},
+		DenyList:    []string{".go"},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
@@ -29,7 +29,7 @@ func TestFileCompile(t *testing.T) {
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile file request")
 
-	require.Contains(t, request.extensionDenylist, ".go", "could not get .go in denylist")
+	require.Contains(t, request.denyList, ".go", "could not get .go in denylist")
 	require.NotContains(t, request.extensions, ".go", "could get .go in allowlist")
 	require.True(t, request.allExtensions, "could not get correct allExtensions")
 }

v2/pkg/protocols/file/find.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/karrick/godirwalk"
 	"github.com/pkg/errors"
-
+	"github.com/projectdiscovery/folderutil"
 	"github.com/projectdiscovery/gologger"
 )
 
@@ -51,7 +51,7 @@ func (request *Request) findGlobPathMatches(absPath string, processed map[string
 		return errors.Errorf("wildcard found, but unable to glob: %s\n", err)
 	}
 	for _, match := range matches {
-		if !request.validatePath(match) {
+		if !request.validatePath(absPath, match) {
 			continue
 		}
 		if _, ok := processed[match]; !ok {
@@ -73,7 +73,7 @@ func (request *Request) findFileMatches(absPath string, processed map[string]str
 		return false, nil
 	}
 	if _, ok := processed[absPath]; !ok {
-		if !request.validatePath(absPath) {
+		if !request.validatePath(absPath, absPath) {
 			return false, nil
 		}
 		processed[absPath] = struct{}{}
@@ -93,7 +93,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin
 			if d.IsDir() {
 				return nil
 			}
-			if !request.validatePath(path) {
+			if !request.validatePath(absPath, path) {
 				return nil
 			}
 			if _, ok := processed[path]; !ok {
@@ -107,7 +107,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin
 }
 
 // validatePath validates a file path for blacklist and whitelist options
-func (request *Request) validatePath(item string) bool {
+func (request *Request) validatePath(absPath, item string) bool {
 	extension := filepath.Ext(item)
 
 	if len(request.extensions) > 0 {
@@ -117,9 +117,81 @@ func (request *Request) validatePath(item string) bool {
 			return false
 		}
 	}
-	if _, ok := request.extensionDenylist[extension]; ok {
-		gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, extension)
+	if matchingRule, ok := request.isInDenyList(absPath, item); ok {
+		gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, matchingRule)
 		return false
 	}
+
 	return true
 }
+
+func (request *Request) isInDenyList(absPath, item string) (string, bool) {
+	extension := filepath.Ext(item)
+	// check for possible deny rules
+	// - extension is in deny list
+	if _, ok := request.denyList[extension]; ok {
+		return extension, true
+	}
+
+	// - full path is in deny list
+	if _, ok := request.denyList[item]; ok {
+		return item, true
+	}
+
+	// file is in a forbidden subdirectory
+	filename := filepath.Base(item)
+	fullPathWithoutFilename := strings.TrimSuffix(item, filename)
+	relativePathWithFilename := strings.TrimPrefix(item, absPath)
+	relativePath := strings.TrimSuffix(relativePathWithFilename, filename)
+
+	// - filename is in deny list
+	if _, ok := request.denyList[filename]; ok {
+		return filename, true
+	}
+
+	// - relative path is in deny list
+	if _, ok := request.denyList[relativePath]; ok {
+		return relativePath, true
+	}
+
+	// relative path + filename are in the forbidden list
+	if _, ok := request.denyList[relativePathWithFilename]; ok {
+		return relativePathWithFilename, true
+	}
+
+	// root path + relative path are in the forbidden list
+	if _, ok := request.denyList[fullPathWithoutFilename]; ok {
+		return fullPathWithoutFilename, true
+	}
+
+	// check any progressive combined part of the relative and absolute path with filename for matches within rules prefixes
+	if pathTreeItem, ok := request.isAnyChunkInDenyList(relativePath, false); ok {
+		return pathTreeItem, true
+	}
+	if pathTreeItem, ok := request.isAnyChunkInDenyList(item, true); ok {
+		return pathTreeItem, true
+	}
+
+	return "", false
+}
+
+func (request *Request) isAnyChunkInDenyList(path string, splitWithUtils bool) (string, bool) {
+	var paths []string
+
+	if splitWithUtils {
+		pathInfo, _ := folderutil.NewPathInfo(path)
+		paths, _ = pathInfo.Paths()
+	} else {
+		pathTree := strings.Split(path, string(os.PathSeparator))
+		for i := range pathTree {
+			paths = append(paths, filepath.Join(pathTree[:i]...))
+		}
+	}
+	for _, pathTreeItem := range paths {
+		if _, ok := request.denyList[pathTreeItem]; ok {
+			return pathTreeItem, true
+		}
+	}
+
+	return "", false
+}

v2/pkg/protocols/file/find_test.go

@@ -19,11 +19,11 @@ func TestFindInputPaths(t *testing.T) {
 	testutils.Init(options)
 	templateID := "testing-file"
 	request := &Request{
-		ID:                templateID,
-		MaxSize:           1024,
-		NoRecursive:       false,
-		Extensions:        []string{"all", ".lock"},
-		ExtensionDenylist: []string{".go"},
+		ID:          templateID,
+		MaxSize:     1024,
+		NoRecursive: false,
+		Extensions:  []string{"all", ".lock"},
+		DenyList:    []string{".go"},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,

v2/pkg/protocols/file/operators_test.go

@@ -20,11 +20,11 @@ func TestResponseToDSLMap(t *testing.T) {
 	testutils.Init(options)
 	templateID := "testing-file"
 	request := &Request{
-		ID:                templateID,
-		MaxSize:           1024,
-		NoRecursive:       false,
-		Extensions:        []string{"*", ".lock"},
-		ExtensionDenylist: []string{".go"},
+		ID:          templateID,
+		MaxSize:     1024,
+		NoRecursive: false,
+		Extensions:  []string{"*", ".lock"},
+		DenyList:    []string{".go"},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
@@ -45,11 +45,11 @@ func TestFileOperatorMatch(t *testing.T) {
 	testutils.Init(options)
 	templateID := "testing-file"
 	request := &Request{
-		ID:                templateID,
-		MaxSize:           1024,
-		NoRecursive:       false,
-		Extensions:        []string{"*", ".lock"},
-		ExtensionDenylist: []string{".go"},
+		ID:          templateID,
+		MaxSize:     1024,
+		NoRecursive: false,
+		Extensions:  []string{"*", ".lock"},
+		DenyList:    []string{".go"},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
@@ -133,11 +133,11 @@ func TestFileOperatorExtract(t *testing.T) {
 	testutils.Init(options)
 	templateID := "testing-file"
 	request := &Request{
-		ID:                templateID,
-		MaxSize:           1024,
-		NoRecursive:       false,
-		Extensions:        []string{"*", ".lock"},
-		ExtensionDenylist: []string{".go"},
+		ID:          templateID,
+		MaxSize:     1024,
+		NoRecursive: false,
+		Extensions:  []string{"*", ".lock"},
+		DenyList:    []string{".go"},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
@@ -240,11 +240,11 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi
 	testutils.Init(options)
 	templateID := "testing-file"
 	request := &Request{
-		ID:                templateID,
-		MaxSize:           1024,
-		NoRecursive:       false,
-		Extensions:        []string{"*", ".lock"},
-		ExtensionDenylist: []string{".go"},
+		ID:          templateID,
+		MaxSize:     1024,
+		NoRecursive: false,
+		Extensions:  []string{"*", ".lock"},
+		DenyList:    []string{".go"},
 		Operators: operators.Operators{
 			MatchersCondition: matcherCondition,
 			Matchers:          matchers,

v2/pkg/protocols/file/request_test.go

@@ -23,11 +23,11 @@ func TestFileExecuteWithResults(t *testing.T) {
 	testutils.Init(options)
 	templateID := "testing-file"
 	request := &Request{
-		ID:                templateID,
-		MaxSize:           1024,
-		NoRecursive:       false,
-		Extensions:        []string{"all"},
-		ExtensionDenylist: []string{".go"},
+		ID:          templateID,
+		MaxSize:     1024,
+		NoRecursive: false,
+		Extensions:  []string{"all"},
+		DenyList:    []string{".go"},
 		Operators: operators.Operators{
 			Matchers: []*matchers.Matcher{{
 				Name:  "test",