json templates support (load with flags, run & validate ) (#3424)

* extending template identification logic

* removing test code

* local debug

* json template loading support using flags

* blacklist meta json files

* minor changes

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>

v2/cmd/sign-templates/main.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/goflags"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates/signer"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
@@ -90,7 +91,7 @@ func processItem(sign *signer.Signer, item string) error {
 
 func processFile(sign *signer.Signer, filePath string) error {
 	ext := filepath.Ext(filePath)
-	if !stringsutil.EqualFoldAny(ext, ".yaml") {
+	if !stringsutil.EqualFoldAny(ext, extensions.YAML) {
 		return nil
 	}
 	err := signTemplate(sign, filePath)

v2/go.mod

@@ -81,7 +81,7 @@ require (
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.0.6
 	github.com/projectdiscovery/uncover v1.0.2
-	github.com/projectdiscovery/utils v0.0.15
+	github.com/projectdiscovery/utils v0.0.16
 	github.com/projectdiscovery/wappalyzergo v0.0.81
 	github.com/stretchr/testify v1.8.2
 	gopkg.in/src-d/go-git.v4 v4.13.1
@@ -195,7 +195,7 @@ require (
 	go.uber.org/zap v1.24.0 // indirect
 	goftp.io/server/v2 v2.0.0 // indirect
 	golang.org/x/crypto v0.7.0
-	golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2
+	golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0
 	golang.org/x/mod v0.9.0 // indirect
 	golang.org/x/sys v0.6.0 // indirect
 	golang.org/x/time v0.3.0 // indirect

v2/go.sum

@@ -434,8 +434,8 @@ github.com/projectdiscovery/tlsx v1.0.6 h1:omMbtedk4BjXtauPpB9Y+FQml9cVthOnIxOMK
 github.com/projectdiscovery/tlsx v1.0.6/go.mod h1:9PTwYVVbaLYpNIwZIvgVxJzctbiemM/pgukkOb3/4wY=
 github.com/projectdiscovery/uncover v1.0.2 h1:mRFzflYyvwKkHd3XKufMlDRrb6p1mjFZTSHoNAUpFwo=
 github.com/projectdiscovery/uncover v1.0.2/go.mod h1:lz4QYfArSA6jJkXyB71kN2/Pc7IW7nJB8c95n7xtwqY=
-github.com/projectdiscovery/utils v0.0.15 h1:32RMOryDiwT+OyPMaMR9LDYcBliUD3aOqvFVzkEsyWI=
-github.com/projectdiscovery/utils v0.0.15/go.mod h1:2CyxZXcx62NUiGJZZam23CpphqXy3kaomE9uvgHgkEo=
+github.com/projectdiscovery/utils v0.0.16 h1:7vmi3haCyM3vk0yXSLjoid4p2/7bo042rcmG4Dtk+Sk=
+github.com/projectdiscovery/utils v0.0.16/go.mod h1:Cu216AlQ7rAYa8aDBqB2OgNfu5p24Uj+tG9RxV8Wbfs=
 github.com/projectdiscovery/wappalyzergo v0.0.81 h1:i7WYrH+O2EoHbY1g/WnrxO4YF/0OkA/G1bw6z8WKcjA=
 github.com/projectdiscovery/wappalyzergo v0.0.81/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@@ -614,8 +614,8 @@ golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 h1:Jvc7gsqn21cJHCmAWx0LiimpP18LZmUxkT5Mp7EZ1mI=
-golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
+golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0 h1:LGJsf5LRplCck6jUCH3dBL2dmycNruWNF5xugkSlfXw=
+golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=

v2/internal/runner/cloud.go

@@ -15,6 +15,7 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/internal/runner/nucleicloud"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions"
 )
 
 // Get all the scan lists for a user/apikey.
@@ -266,7 +267,7 @@ func (r *Runner) addTemplate(location string) error {
 		if err != nil {
 			return err
 		}
-		if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".yaml") {
+		if d.IsDir() || !strings.EqualFold(filepath.Ext(path), extensions.YAML) {
 			return nil
 		}
 		base := filepath.Base(path)
@@ -337,7 +338,7 @@ func (r *Runner) removeTemplate(item string) error {
 	var err error
 	if ID, parseErr := strconv.ParseInt(item, 10, 64); parseErr == nil {
 		err = r.cloudClient.RemoveTemplate(ID, "")
-	} else if strings.EqualFold(path.Ext(item), ".yaml") {
+	} else if strings.EqualFold(path.Ext(item), extensions.YAML) {
 		err = r.cloudClient.RemoveTemplate(0, item)
 	} else {
 		return r.removeTemplatePrefix(item)

v2/internal/runner/runner.go

@@ -792,8 +792,10 @@ func (r *Runner) countNewTemplates() int {
 	return count
 }
 
+// isTemplate is a callback function used by goflags to decide if given file should be read
+// if it is not a nuclei-template file only then file is read
 func isTemplate(filename string) bool {
-	return stringsutil.EqualFoldAny(filepath.Ext(filename), templates.TemplateExtension)
+	return stringsutil.EqualFoldAny(filepath.Ext(filename), config.GetSupportTemplateFileExtensions()...)
 }
 
 // SaveResumeConfig to file

v2/pkg/catalog/config/template.go

@@ -0,0 +1,35 @@
+package config
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions"
+)
+
+// TemplateFormat
+type TemplateFormat uint8
+
+const (
+	YAML TemplateFormat = iota
+	JSON
+	Unknown
+)
+
+// GetTemplateFormatFromExt returns template format
+func GetTemplateFormatFromExt(filePath string) TemplateFormat {
+	fileExt := strings.ToLower(filepath.Ext(filePath))
+	switch fileExt {
+	case extensions.JSON:
+		return JSON
+	case extensions.YAML:
+		return YAML
+	default:
+		return Unknown
+	}
+}
+
+// GetSupportedTemplateFileExtensions returns all supported template file extensions
+func GetSupportTemplateFileExtensions() []string {
+	return []string{extensions.YAML, extensions.JSON}
+}

v2/pkg/catalog/disk/find.go

@@ -2,11 +2,14 @@ package disk
 
 import (
 	"io/fs"
+	"log"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/pkg/errors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
 // GetTemplatesPath returns a list of absolute paths for the provided template list.
@@ -16,8 +19,14 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri
 	allTemplates := []string{}
 	erred := make(map[string]error)
 
+	log.Println(definitions)
 	for _, t := range definitions {
-		if strings.HasPrefix(t, "http") && (strings.HasSuffix(t, ".yaml") || strings.HasSuffix(t, ".yml")) {
+		if stringsutil.ContainsAny(t, knownConfigFiles...) {
+			// TODO: this is a temporary fix to avoid treating these files as templates
+			// this should be replaced with more appropriate and robust logic
+			continue
+		}
+		if strings.HasPrefix(t, "http") && stringsutil.ContainsAny(t, config.GetSupportTemplateFileExtensions()...) {
 			if _, ok := processed[t]; !ok {
 				processed[t] = true
 				allTemplates = append(allTemplates, t)
@@ -35,7 +44,17 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri
 			}
 		}
 	}
-	return allTemplates, erred
+	// purge all falsepositivies
+	filteredTemplates := []string{}
+	for _, v := range allTemplates {
+		// TODO: this is a temporary fix to avoid treating these files as templates
+		// this should be replaced with more appropriate and robust logic
+		if !stringsutil.ContainsAny(v, knownConfigFiles...) {
+			filteredTemplates = append(filteredTemplates, v)
+		}
+	}
+
+	return filteredTemplates, erred
 }
 
 // GetTemplatePath parses the specified input template path and returns a compiled
@@ -43,7 +62,6 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri
 // or folders provided as in.
 func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) {
 	processed := make(map[string]struct{})
-
 	absPath, err := c.convertPathToAbsolute(target)
 	if err != nil {
 		return nil, errors.Wrapf(err, "could not find template file")
@@ -142,7 +160,7 @@ func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]
 			if err != nil {
 				return nil
 			}
-			if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
+			if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown {
 				if _, ok := processed[path]; !ok {
 					results = append(results, path)
 					processed[path] = struct{}{}

v2/pkg/catalog/disk/known-files.go

@@ -0,0 +1,3 @@
+package disk
+
+var knownConfigFiles = []string{"cves.json", "contributors.json", "TEMPLATES-STATS.json"}

v2/pkg/catalog/loader/remote_loader.go

@@ -8,8 +8,10 @@ import (
 
 	"github.com/pkg/errors"
 
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions"
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
 	"github.com/projectdiscovery/retryablehttp-go"
+	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
 type ContentType string
@@ -65,7 +67,7 @@ func getRemoteContent(URL string, remoteTemplateDomainList []string, remoteConte
 		}
 		return
 	}
-	if strings.HasPrefix(URL, "http") && (strings.HasSuffix(URL, ".yaml") || strings.HasSuffix(URL, ".yml")) {
+	if strings.HasPrefix(URL, "http") && stringsutil.HasSuffixAny(URL, extensions.YAML, extensions.YML) {
 		remoteContentChannel <- RemoteContent{
 			Content: []string{URL},
 			Type:    contentType,

v2/pkg/parsers/parser.go

@@ -1,11 +1,13 @@
 package parsers
 
 import (
+	"encoding/json"
 	"fmt"
 	"regexp"
 	"strings"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates/cache"
@@ -130,11 +132,18 @@ func ParseTemplate(templatePath string, catalog catalog.Catalog) (*templates.Tem
 		template.Verified, _ = signer.Verify(signer.DefaultVerifier, data)
 	}
 
+	switch config.GetTemplateFormatFromExt(templatePath) {
+	case config.JSON:
+		err = json.Unmarshal(data, template)
+	case config.YAML:
 		if NoStrictSyntax {
 			err = yaml.Unmarshal(data, template)
 		} else {
 			err = yaml.UnmarshalStrict(data, template)
 		}
+	default:
+		err = fmt.Errorf("failed to identify template format expected JSON or YAML but got %v", templatePath)
+	}
 	if err != nil {
 		stats.Increment(SyntaxErrorStats)
 		return nil, err

v2/pkg/templates/extensions/extensions.go

@@ -0,0 +1,7 @@
+package extensions
+
+const (
+	JSON = ".json"
+	YAML = ".yaml"
+	YML  = ".yml"
+)

v2/pkg/templates/templates.go

@@ -22,11 +22,6 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-const (
-	// TemplateExtension defines the template default file extension
-	TemplateExtension = ".yaml"
-)
-
 // Template is a YAML input file which defines all the requests and
 // other metadata for a template.
 type Template struct {

v2/pkg/utils/utils.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils/yaml"
 	"github.com/projectdiscovery/retryablehttp-go"
 	fileutil "github.com/projectdiscovery/utils/file"
@@ -58,7 +59,7 @@ func ReadFromPathOrURL(templatePath string, catalog catalog.Catalog) (data []byt
 	}
 
 	// pre-process directives only for local files
-	if fileutil.FileExists(templatePath) {
+	if fileutil.FileExists(templatePath) && config.GetTemplateFormatFromExt(templatePath) == config.YAML {
 		data, err = yaml.PreProcess(data)
 		if err != nil {
 			return nil, err

v2/pkg/utils/yaml/preprocess.go

@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions"
 	fileutil "github.com/projectdiscovery/utils/file"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
@@ -52,7 +53,7 @@ func PreProcess(data []byte) ([]byte, error) {
 				return nil, err
 			}
 			// if it's yaml, tries to preprocess that too recursively
-			if stringsutil.HasSuffixAny(includeFileName, ".yaml") {
+			if stringsutil.HasSuffixAny(includeFileName, extensions.YAML) {
 				if subIncludedFileContent, err := PreProcess(includeFileContent); err == nil {
 					includeFileContent = subIncludedFileContent
 				} else {