diff --git a/v2/cmd/sign-templates/main.go b/v2/cmd/sign-templates/main.go index 3c1e800c..964d5ce3 100644 --- a/v2/cmd/sign-templates/main.go +++ b/v2/cmd/sign-templates/main.go @@ -8,6 +8,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions" "github.com/projectdiscovery/nuclei/v2/pkg/templates/signer" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -90,7 +91,7 @@ func processItem(sign *signer.Signer, item string) error { func processFile(sign *signer.Signer, filePath string) error { ext := filepath.Ext(filePath) - if !stringsutil.EqualFoldAny(ext, ".yaml") { + if !stringsutil.EqualFoldAny(ext, extensions.YAML) { return nil } err := signTemplate(sign, filePath) diff --git a/v2/go.mod b/v2/go.mod index ef124d70..c55b243e 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -81,7 +81,7 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.0.6 github.com/projectdiscovery/uncover v1.0.2 - github.com/projectdiscovery/utils v0.0.15 + github.com/projectdiscovery/utils v0.0.16 github.com/projectdiscovery/wappalyzergo v0.0.81 github.com/stretchr/testify v1.8.2 gopkg.in/src-d/go-git.v4 v4.13.1 @@ -195,7 +195,7 @@ require ( go.uber.org/zap v1.24.0 // indirect goftp.io/server/v2 v2.0.0 // indirect golang.org/x/crypto v0.7.0 - golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 + golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0 golang.org/x/mod v0.9.0 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 3b814f56..6837de9e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -434,8 +434,8 @@ github.com/projectdiscovery/tlsx v1.0.6 h1:omMbtedk4BjXtauPpB9Y+FQml9cVthOnIxOMK github.com/projectdiscovery/tlsx v1.0.6/go.mod h1:9PTwYVVbaLYpNIwZIvgVxJzctbiemM/pgukkOb3/4wY= github.com/projectdiscovery/uncover v1.0.2 h1:mRFzflYyvwKkHd3XKufMlDRrb6p1mjFZTSHoNAUpFwo= github.com/projectdiscovery/uncover v1.0.2/go.mod h1:lz4QYfArSA6jJkXyB71kN2/Pc7IW7nJB8c95n7xtwqY= -github.com/projectdiscovery/utils v0.0.15 h1:32RMOryDiwT+OyPMaMR9LDYcBliUD3aOqvFVzkEsyWI= -github.com/projectdiscovery/utils v0.0.15/go.mod h1:2CyxZXcx62NUiGJZZam23CpphqXy3kaomE9uvgHgkEo= +github.com/projectdiscovery/utils v0.0.16 h1:7vmi3haCyM3vk0yXSLjoid4p2/7bo042rcmG4Dtk+Sk= +github.com/projectdiscovery/utils v0.0.16/go.mod h1:Cu216AlQ7rAYa8aDBqB2OgNfu5p24Uj+tG9RxV8Wbfs= github.com/projectdiscovery/wappalyzergo v0.0.81 h1:i7WYrH+O2EoHbY1g/WnrxO4YF/0OkA/G1bw6z8WKcjA= github.com/projectdiscovery/wappalyzergo v0.0.81/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -614,8 +614,8 @@ golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 h1:Jvc7gsqn21cJHCmAWx0LiimpP18LZmUxkT5Mp7EZ1mI= -golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0 h1:LGJsf5LRplCck6jUCH3dBL2dmycNruWNF5xugkSlfXw= +golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= diff --git a/v2/internal/runner/cloud.go b/v2/internal/runner/cloud.go index 54c5fc6d..ce40b30c 100644 --- a/v2/internal/runner/cloud.go +++ b/v2/internal/runner/cloud.go @@ -15,6 +15,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/runner/nucleicloud" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions" ) // Get all the scan lists for a user/apikey. @@ -266,7 +267,7 @@ func (r *Runner) addTemplate(location string) error { if err != nil { return err } - if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".yaml") { + if d.IsDir() || !strings.EqualFold(filepath.Ext(path), extensions.YAML) { return nil } base := filepath.Base(path) @@ -337,7 +338,7 @@ func (r *Runner) removeTemplate(item string) error { var err error if ID, parseErr := strconv.ParseInt(item, 10, 64); parseErr == nil { err = r.cloudClient.RemoveTemplate(ID, "") - } else if strings.EqualFold(path.Ext(item), ".yaml") { + } else if strings.EqualFold(path.Ext(item), extensions.YAML) { err = r.cloudClient.RemoveTemplate(0, item) } else { return r.removeTemplatePrefix(item) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index db0b97c0..e330d60b 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -792,8 +792,10 @@ func (r *Runner) countNewTemplates() int { return count } +// isTemplate is a callback function used by goflags to decide if given file should be read +// if it is not a nuclei-template file only then file is read func isTemplate(filename string) bool { - return stringsutil.EqualFoldAny(filepath.Ext(filename), templates.TemplateExtension) + return stringsutil.EqualFoldAny(filepath.Ext(filename), config.GetSupportTemplateFileExtensions()...) } // SaveResumeConfig to file diff --git a/v2/pkg/catalog/config/template.go b/v2/pkg/catalog/config/template.go new file mode 100644 index 00000000..b36bd777 --- /dev/null +++ b/v2/pkg/catalog/config/template.go @@ -0,0 +1,35 @@ +package config + +import ( + "path/filepath" + "strings" + + "github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions" +) + +// TemplateFormat +type TemplateFormat uint8 + +const ( + YAML TemplateFormat = iota + JSON + Unknown +) + +// GetTemplateFormatFromExt returns template format +func GetTemplateFormatFromExt(filePath string) TemplateFormat { + fileExt := strings.ToLower(filepath.Ext(filePath)) + switch fileExt { + case extensions.JSON: + return JSON + case extensions.YAML: + return YAML + default: + return Unknown + } +} + +// GetSupportedTemplateFileExtensions returns all supported template file extensions +func GetSupportTemplateFileExtensions() []string { + return []string{extensions.YAML, extensions.JSON} +} diff --git a/v2/pkg/catalog/disk/find.go b/v2/pkg/catalog/disk/find.go index 9c1dff46..b6702965 100644 --- a/v2/pkg/catalog/disk/find.go +++ b/v2/pkg/catalog/disk/find.go @@ -2,11 +2,14 @@ package disk import ( "io/fs" + "log" "os" "path/filepath" "strings" "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + stringsutil "github.com/projectdiscovery/utils/strings" ) // GetTemplatesPath returns a list of absolute paths for the provided template list. @@ -16,8 +19,14 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri allTemplates := []string{} erred := make(map[string]error) + log.Println(definitions) for _, t := range definitions { - if strings.HasPrefix(t, "http") && (strings.HasSuffix(t, ".yaml") || strings.HasSuffix(t, ".yml")) { + if stringsutil.ContainsAny(t, knownConfigFiles...) { + // TODO: this is a temporary fix to avoid treating these files as templates + // this should be replaced with more appropriate and robust logic + continue + } + if strings.HasPrefix(t, "http") && stringsutil.ContainsAny(t, config.GetSupportTemplateFileExtensions()...) { if _, ok := processed[t]; !ok { processed[t] = true allTemplates = append(allTemplates, t) @@ -35,7 +44,17 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri } } } - return allTemplates, erred + // purge all falsepositivies + filteredTemplates := []string{} + for _, v := range allTemplates { + // TODO: this is a temporary fix to avoid treating these files as templates + // this should be replaced with more appropriate and robust logic + if !stringsutil.ContainsAny(v, knownConfigFiles...) { + filteredTemplates = append(filteredTemplates, v) + } + } + + return filteredTemplates, erred } // GetTemplatePath parses the specified input template path and returns a compiled @@ -43,7 +62,6 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri // or folders provided as in. func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) { processed := make(map[string]struct{}) - absPath, err := c.convertPathToAbsolute(target) if err != nil { return nil, errors.Wrapf(err, "could not find template file") @@ -142,7 +160,7 @@ func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string] if err != nil { return nil } - if !d.IsDir() && strings.HasSuffix(path, ".yaml") { + if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown { if _, ok := processed[path]; !ok { results = append(results, path) processed[path] = struct{}{} diff --git a/v2/pkg/catalog/disk/known-files.go b/v2/pkg/catalog/disk/known-files.go new file mode 100644 index 00000000..6660363a --- /dev/null +++ b/v2/pkg/catalog/disk/known-files.go @@ -0,0 +1,3 @@ +package disk + +var knownConfigFiles = []string{"cves.json", "contributors.json", "TEMPLATES-STATS.json"} diff --git a/v2/pkg/catalog/loader/remote_loader.go b/v2/pkg/catalog/loader/remote_loader.go index 531417ae..2c786168 100644 --- a/v2/pkg/catalog/loader/remote_loader.go +++ b/v2/pkg/catalog/loader/remote_loader.go @@ -8,8 +8,10 @@ import ( "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/retryablehttp-go" + stringsutil "github.com/projectdiscovery/utils/strings" ) type ContentType string @@ -65,7 +67,7 @@ func getRemoteContent(URL string, remoteTemplateDomainList []string, remoteConte } return } - if strings.HasPrefix(URL, "http") && (strings.HasSuffix(URL, ".yaml") || strings.HasSuffix(URL, ".yml")) { + if strings.HasPrefix(URL, "http") && stringsutil.HasSuffixAny(URL, extensions.YAML, extensions.YML) { remoteContentChannel <- RemoteContent{ Content: []string{URL}, Type: contentType, diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 6c8c6cc1..23c886e7 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -1,11 +1,13 @@ package parsers import ( + "encoding/json" "fmt" "regexp" "strings" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates/cache" @@ -130,10 +132,17 @@ func ParseTemplate(templatePath string, catalog catalog.Catalog) (*templates.Tem template.Verified, _ = signer.Verify(signer.DefaultVerifier, data) } - if NoStrictSyntax { - err = yaml.Unmarshal(data, template) - } else { - err = yaml.UnmarshalStrict(data, template) + switch config.GetTemplateFormatFromExt(templatePath) { + case config.JSON: + err = json.Unmarshal(data, template) + case config.YAML: + if NoStrictSyntax { + err = yaml.Unmarshal(data, template) + } else { + err = yaml.UnmarshalStrict(data, template) + } + default: + err = fmt.Errorf("failed to identify template format expected JSON or YAML but got %v", templatePath) } if err != nil { stats.Increment(SyntaxErrorStats) diff --git a/v2/pkg/templates/extensions/extensions.go b/v2/pkg/templates/extensions/extensions.go new file mode 100644 index 00000000..f4522b52 --- /dev/null +++ b/v2/pkg/templates/extensions/extensions.go @@ -0,0 +1,7 @@ +package extensions + +const ( + JSON = ".json" + YAML = ".yaml" + YML = ".yml" +) diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 54e5f487..84f67f9b 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -22,11 +22,6 @@ import ( "gopkg.in/yaml.v2" ) -const ( - // TemplateExtension defines the template default file extension - TemplateExtension = ".yaml" -) - // Template is a YAML input file which defines all the requests and // other metadata for a template. type Template struct { diff --git a/v2/pkg/utils/utils.go b/v2/pkg/utils/utils.go index ba04589a..9b6e42c1 100644 --- a/v2/pkg/utils/utils.go +++ b/v2/pkg/utils/utils.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/utils/yaml" "github.com/projectdiscovery/retryablehttp-go" fileutil "github.com/projectdiscovery/utils/file" @@ -58,7 +59,7 @@ func ReadFromPathOrURL(templatePath string, catalog catalog.Catalog) (data []byt } // pre-process directives only for local files - if fileutil.FileExists(templatePath) { + if fileutil.FileExists(templatePath) && config.GetTemplateFormatFromExt(templatePath) == config.YAML { data, err = yaml.PreProcess(data) if err != nil { return nil, err diff --git a/v2/pkg/utils/yaml/preprocess.go b/v2/pkg/utils/yaml/preprocess.go index 6dd0e121..c918f07b 100644 --- a/v2/pkg/utils/yaml/preprocess.go +++ b/v2/pkg/utils/yaml/preprocess.go @@ -7,6 +7,7 @@ import ( "regexp" "strings" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions" fileutil "github.com/projectdiscovery/utils/file" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -52,7 +53,7 @@ func PreProcess(data []byte) ([]byte, error) { return nil, err } // if it's yaml, tries to preprocess that too recursively - if stringsutil.HasSuffixAny(includeFileName, ".yaml") { + if stringsutil.HasSuffixAny(includeFileName, extensions.YAML) { if subIncludedFileContent, err := PreProcess(includeFileContent); err == nil { includeFileContent = subIncludedFileContent } else {