Merge pull request #1746 from projectdiscovery/issue-1703-minor-changes

Stats counter + Mime type filter

integration_tests/headless/headless-extract-values.yaml

@@ -17,7 +17,7 @@ headless:
         name: extract
         args:
           code: |
-            '\n' + [...new Set(Array.from(document.querySelectorAll('[src], [href], [url], [action]')).map(i => i.src || i.href || i.url || i.action))].join('\r\n') + '\n'
+            () => '\n' + [...new Set(Array.from(document.querySelectorAll('[src], [href], [url], [action]')).map(i => i.src || i.href || i.url || i.action))].join('\r\n') + '\n'
 
     matchers:
       - type: word

v2/go.mod

@@ -114,6 +114,7 @@ require (
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gosuri/uilive v0.0.4 // indirect
 	github.com/gosuri/uiprogress v0.0.1 // indirect
+	github.com/h2non/filetype v1.1.3 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
 	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
 	github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect

v2/go.sum

@@ -263,6 +263,8 @@ github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY=
 github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI=
 github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw=
 github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0=
+github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
+github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
 github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=

v2/pkg/protocols/file/file.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"github.com/docker/go-units"
+	"github.com/h2non/filetype"
 	"github.com/pkg/errors"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
@@ -21,12 +22,12 @@ type Request struct {
 	// Operators for the current request go here.
 	operators.Operators `yaml:",inline"`
 	// description: |
-	//   Extensions is the list of extensions to perform matching on.
+	//   Extensions is the list of extensions or mime types to perform matching on.
 	// examples:
 	//   - value: '[]string{".txt", ".go", ".json"}'
 	Extensions []string `yaml:"extensions,omitempty" jsonschema:"title=extensions to match,description=List of extensions to perform matching on"`
 	// description: |
-	//   DenyList is the list of file, directories or extensions to deny during matching.
+	//   DenyList is the list of file, directories, mime types or extensions to deny during matching.
 	//
 	//   By default, it contains some non-interesting extensions that are hardcoded
 	//   in nuclei.
@@ -52,12 +53,18 @@ type Request struct {
 	//   elaborates archives
 	Archive bool
 
+	// description: |
+	//   enables mime types check
+	MimeType bool
+
 	CompiledOperators *operators.Operators `yaml:"-"`
 
 	// cache any variables that may be needed for operation.
-	options    *protocols.ExecuterOptions
-	extensions map[string]struct{}
-	denyList   map[string]struct{}
+	options             *protocols.ExecuterOptions
+	mimeTypesChecks     []string
+	extensions          map[string]struct{}
+	denyList            map[string]struct{}
+	denyMimeTypesChecks []string
 
 	// description: |
 	//   NoRecursive specifies whether to not do recursive checks if folders are provided.
@@ -120,15 +127,20 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 	request.denyList = make(map[string]struct{})
 
 	for _, extension := range request.Extensions {
-		if extension == "all" {
+		switch {
+		case extension == "all":
 			request.allExtensions = true
-		} else {
+		case request.MimeType && filetype.IsMIMESupported(extension):
+			continue
+		default:
 			if !strings.HasPrefix(extension, ".") {
 				extension = "." + extension
 			}
 			request.extensions[extension] = struct{}{}
 		}
 	}
+	request.mimeTypesChecks = extractMimeTypes(request.Extensions)
+
 	// process default denylist (extensions)
 	var denyList []string
 	if !request.Archive {
@@ -147,9 +159,30 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 		// also add a cleaned version as the exclusion path can be dirty (eg. /a/b/c, /a/b/c/, a///b///c/../d)
 		request.denyList[filepath.Clean(excludeItem)] = struct{}{}
 	}
+	request.denyMimeTypesChecks = extractMimeTypes(request.DenyList)
 	return nil
 }
 
+func matchAnyMimeTypes(data []byte, mimeTypes []string) bool {
+	for _, mimeType := range mimeTypes {
+		if filetype.Is(data, mimeType) {
+			return true
+		}
+	}
+	return false
+}
+
+func extractMimeTypes(m []string) []string {
+	var mimeTypes []string
+	for _, mm := range m {
+		if !filetype.IsMIMESupported(mm) {
+			continue
+		}
+		mimeTypes = append(mimeTypes, mm)
+	}
+	return mimeTypes
+}
+
 // Requests returns the total number of requests the YAML rule will perform
 func (request *Request) Requests() int {
 	return 0

v2/pkg/protocols/file/find.go

@@ -1,12 +1,14 @@
 package file
 
 import (
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/karrick/godirwalk"
 	"github.com/pkg/errors"
+	"github.com/projectdiscovery/fileutil"
 	"github.com/projectdiscovery/folderutil"
 	"github.com/projectdiscovery/gologger"
 )
@@ -51,7 +53,7 @@ func (request *Request) findGlobPathMatches(absPath string, processed map[string
 		return errors.Errorf("wildcard found, but unable to glob: %s\n", err)
 	}
 	for _, match := range matches {
-		if !request.validatePath(absPath, match) {
+		if !request.validatePath(absPath, match, false) {
 			continue
 		}
 		if _, ok := processed[match]; !ok {
@@ -73,7 +75,7 @@ func (request *Request) findFileMatches(absPath string, processed map[string]str
 		return false, nil
 	}
 	if _, ok := processed[absPath]; !ok {
-		if !request.validatePath(absPath, absPath) {
+		if !request.validatePath(absPath, absPath, false) {
 			return false, nil
 		}
 		processed[absPath] = struct{}{}
@@ -93,7 +95,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin
 			if d.IsDir() {
 				return nil
 			}
-			if !request.validatePath(absPath, path) {
+			if !request.validatePath(absPath, path, false) {
 				return nil
 			}
 			if _, ok := processed[path]; !ok {
@@ -107,9 +109,9 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin
 }
 
 // validatePath validates a file path for blacklist and whitelist options
-func (request *Request) validatePath(absPath, item string) bool {
+func (request *Request) validatePath(absPath, item string, inArchive bool) bool {
 	extension := filepath.Ext(item)
-
+	// extension check
 	if len(request.extensions) > 0 {
 		if _, ok := request.extensions[extension]; ok {
 			return true
@@ -117,11 +119,35 @@ func (request *Request) validatePath(absPath, item string) bool {
 			return false
 		}
 	}
+
+	var (
+		fileExists bool
+		dataChunk  []byte
+	)
+	if !inArchive && request.MimeType {
+		// mime type check
+		// read first bytes to infer runtime type
+		fileExists = fileutil.FileExists(item)
+		if fileExists {
+			dataChunk, _ = readChunk(item)
+			if len(request.mimeTypesChecks) > 0 && matchAnyMimeTypes(dataChunk, request.mimeTypesChecks) {
+				return true
+			}
+		}
+	}
+
 	if matchingRule, ok := request.isInDenyList(absPath, item); ok {
 		gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, matchingRule)
 		return false
 	}
 
+	// denied mime type checks
+	if !inArchive && request.MimeType && fileExists {
+		if len(request.denyMimeTypesChecks) > 0 && matchAnyMimeTypes(dataChunk, request.denyMimeTypesChecks) {
+			return false
+		}
+	}
+
 	return true
 }
 
@@ -175,6 +201,21 @@ func (request *Request) isInDenyList(absPath, item string) (string, bool) {
 	return "", false
 }
 
+func readChunk(fileName string) ([]byte, error) {
+	r, err := os.Open(fileName)
+	if err != nil {
+		return nil, err
+	}
+
+	defer r.Close()
+
+	var buff [1024]byte
+	if _, err = io.ReadFull(r, buff[:]); err != nil {
+		return nil, err
+	}
+	return buff[:], nil
+}
+
 func (request *Request) isAnyChunkInDenyList(path string, splitWithUtils bool) (string, bool) {
 	var paths []string
 

v2/pkg/protocols/file/request.go

@@ -46,7 +46,6 @@ var emptyResultErr = errors.New("Empty result")
 func (request *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	wg := sizedwaitgroup.New(request.options.Options.BulkSize)
 	err := request.getInputPaths(input, func(filePath string) {
-		request.options.Progress.AddToTotal(1)
 		wg.Add()
 		func(filePath string) {
 			defer wg.Done()
@@ -56,20 +55,28 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 				switch archiveInstance := archiveReader.(type) {
 				case archiver.Walker:
 					err := archiveInstance.Walk(filePath, func(file archiver.File) error {
-						if !request.validatePath("/", file.Name()) {
+						if !request.validatePath("/", file.Name(), true) {
 							return nil
 						}
+						// every new file in the compressed multi-file archive counts 1
+						request.options.Progress.AddToTotal(1)
 						archiveFileName := filepath.Join(filePath, file.Name())
 						event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input, file.Size(), previous)
 						if err != nil {
 							if errors.Is(err, emptyResultErr) {
+								// no matches but one file elaborated
+								request.options.Progress.IncrementRequests()
 								return nil
 							}
+							gologger.Error().Msgf("%s\n", err)
+							// error while elaborating the file
+							request.options.Progress.IncrementFailedRequestsBy(1)
 							return err
 						}
 						defer file.Close()
 						dumpResponse(event, request.options, fileMatches, filePath)
 						callback(event)
+						// file elaborated and matched
 						request.options.Progress.IncrementRequests()
 						return nil
 					})
@@ -78,9 +85,13 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 						return
 					}
 				case archiver.Decompressor:
+					// compressed archive - contains only one file => increments the counter by 1
+					request.options.Progress.AddToTotal(1)
 					file, err := os.Open(filePath)
 					if err != nil {
 						gologger.Error().Msgf("%s\n", err)
+						// error while elaborating the file
+						request.options.Progress.IncrementFailedRequestsBy(1)
 						return
 					}
 					defer file.Close()
@@ -88,12 +99,16 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 					tmpFileOut, err := os.CreateTemp("", "")
 					if err != nil {
 						gologger.Error().Msgf("%s\n", err)
+						// error while elaborating the file
+						request.options.Progress.IncrementFailedRequestsBy(1)
 						return
 					}
 					defer tmpFileOut.Close()
 					defer os.RemoveAll(tmpFileOut.Name())
 					if err := archiveInstance.Decompress(file, tmpFileOut); err != nil {
 						gologger.Error().Msgf("%s\n", err)
+						// error while elaborating the file
+						request.options.Progress.IncrementFailedRequestsBy(1)
 						return
 					}
 					_ = tmpFileOut.Sync()
@@ -101,26 +116,39 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 					_, _ = tmpFileOut.Seek(0, 0)
 					event, fileMatches, err := request.processReader(tmpFileOut, filePath, input, fileStat.Size(), previous)
 					if err != nil {
-						if !errors.Is(err, emptyResultErr) {
-							gologger.Error().Msgf("%s\n", err)
+						if errors.Is(err, emptyResultErr) {
+							// no matches but one file elaborated
+							request.options.Progress.IncrementRequests()
+							return
 						}
+						gologger.Error().Msgf("%s\n", err)
+						// error while elaborating the file
+						request.options.Progress.IncrementFailedRequestsBy(1)
 						return
 					}
 					dumpResponse(event, request.options, fileMatches, filePath)
 					callback(event)
+					// file elaborated and matched
 					request.options.Progress.IncrementRequests()
 				}
 			default:
-				// normal file
+				// normal file - increments the counter by 1
+				request.options.Progress.AddToTotal(1)
 				event, fileMatches, err := request.processFile(filePath, input, previous)
 				if err != nil {
-					if !errors.Is(err, emptyResultErr) {
-						gologger.Error().Msgf("%s\n", err)
+					if errors.Is(err, emptyResultErr) {
+						// no matches but one file elaborated
+						request.options.Progress.IncrementRequests()
+						return
 					}
+					gologger.Error().Msgf("%s\n", err)
+					// error while elaborating the file
+					request.options.Progress.IncrementFailedRequestsBy(1)
 					return
 				}
 				dumpResponse(event, request.options, fileMatches, filePath)
 				callback(event)
+				// file elaborated and matched
 				request.options.Progress.IncrementRequests()
 			}
 		}(filePath)

v2/pkg/protocols/headless/engine/page_actions_test.go

@@ -52,7 +52,7 @@ func TestActionScript(t *testing.T) {
 		actions := []*Action{
 			{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
 			{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
-			{ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "window.test"}},
+			{ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "() => window.test"}},
 		}
 
 		testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) {
@@ -64,10 +64,10 @@ func TestActionScript(t *testing.T) {
 
 	t.Run("hook", func(t *testing.T) {
 		actions := []*Action{
-			{ActionType: ActionTypeHolder{ActionType: ActionScript}, Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}},
+			{ActionType: ActionTypeHolder{ActionType: ActionScript}, Data: map[string]string{"code": "() => window.test = 'some-data';", "hook": "true"}},
 			{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
 			{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
-			{ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "window.test"}},
+			{ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "() => window.test"}},
 		}
 		testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) {
 			require.Nil(t, err, "could not run page actions")