daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

json templates support (load with flags, run & validate ) (#3424) 

* extending template identification logic

* removing test code

* local debug

* json template loading support using flags

* blacklist meta json files

* minor changes

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
dev
Mzack9999 2023-03-16 09:03:59 +01:00 committed by GitHub
parent c9634fae72
commit c182434130
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 101 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
v2/cmd/sign-templates/main.go
View File

@ -8,6 +8,7 @@ import (
"github.com/pkg/errors"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/signer"
stringsutil "github.com/projectdiscovery/utils/strings"
)
@ -90,7 +91,7 @@ func processItem(sign *signer.Signer, item string) error {
func processFile(sign *signer.Signer, filePath string) error {
ext := filepath.Ext(filePath)
if !stringsutil.EqualFoldAny(ext, ".yaml") {
if !stringsutil.EqualFoldAny(ext, extensions.YAML) {
return nil
}
err := signTemplate(sign, filePath)

4
v2/go.mod
View File

@ -81,7 +81,7 @@ require (
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.0.6
github.com/projectdiscovery/uncover v1.0.2
github.com/projectdiscovery/utils v0.0.15
github.com/projectdiscovery/utils v0.0.16
github.com/projectdiscovery/wappalyzergo v0.0.81
github.com/stretchr/testify v1.8.2
gopkg.in/src-d/go-git.v4 v4.13.1
@ -195,7 +195,7 @@ require (
go.uber.org/zap v1.24.0 // indirect
goftp.io/server/v2 v2.0.0 // indirect
golang.org/x/crypto v0.7.0
golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2
golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0
golang.org/x/mod v0.9.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/time v0.3.0 // indirect

8
v2/go.sum
View File

@ -434,8 +434,8 @@ github.com/projectdiscovery/tlsx v1.0.6 h1:omMbtedk4BjXtauPpB9Y+FQml9cVthOnIxOMK
github.com/projectdiscovery/tlsx v1.0.6/go.mod h1:9PTwYVVbaLYpNIwZIvgVxJzctbiemM/pgukkOb3/4wY=
github.com/projectdiscovery/uncover v1.0.2 h1:mRFzflYyvwKkHd3XKufMlDRrb6p1mjFZTSHoNAUpFwo=
github.com/projectdiscovery/uncover v1.0.2/go.mod h1:lz4QYfArSA6jJkXyB71kN2/Pc7IW7nJB8c95n7xtwqY=
github.com/projectdiscovery/utils v0.0.15 h1:32RMOryDiwT+OyPMaMR9LDYcBliUD3aOqvFVzkEsyWI=
github.com/projectdiscovery/utils v0.0.15/go.mod h1:2CyxZXcx62NUiGJZZam23CpphqXy3kaomE9uvgHgkEo=
github.com/projectdiscovery/utils v0.0.16 h1:7vmi3haCyM3vk0yXSLjoid4p2/7bo042rcmG4Dtk+Sk=
github.com/projectdiscovery/utils v0.0.16/go.mod h1:Cu216AlQ7rAYa8aDBqB2OgNfu5p24Uj+tG9RxV8Wbfs=
github.com/projectdiscovery/wappalyzergo v0.0.81 h1:i7WYrH+O2EoHbY1g/WnrxO4YF/0OkA/G1bw6z8WKcjA=
github.com/projectdiscovery/wappalyzergo v0.0.81/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@ -614,8 +614,8 @@ golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 h1:Jvc7gsqn21cJHCmAWx0LiimpP18LZmUxkT5Mp7EZ1mI=
golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0 h1:LGJsf5LRplCck6jUCH3dBL2dmycNruWNF5xugkSlfXw=
golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=

5
v2/internal/runner/cloud.go
View File

@ -15,6 +15,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/runner/nucleicloud"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions"
)
// Get all the scan lists for a user/apikey.
@ -266,7 +267,7 @@ func (r *Runner) addTemplate(location string) error {
if err != nil {
return err
}
if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".yaml") {
if d.IsDir() || !strings.EqualFold(filepath.Ext(path), extensions.YAML) {
return nil
}
base := filepath.Base(path)
@ -337,7 +338,7 @@ func (r *Runner) removeTemplate(item string) error {
var err error
if ID, parseErr := strconv.ParseInt(item, 10, 64); parseErr == nil {
err = r.cloudClient.RemoveTemplate(ID, "")
} else if strings.EqualFold(path.Ext(item), ".yaml") {
} else if strings.EqualFold(path.Ext(item), extensions.YAML) {
err = r.cloudClient.RemoveTemplate(0, item)
} else {
return r.removeTemplatePrefix(item)

4
v2/internal/runner/runner.go
View File

@ -792,8 +792,10 @@ func (r *Runner) countNewTemplates() int {
return count
}
// isTemplate is a callback function used by goflags to decide if given file should be read
// if it is not a nuclei-template file only then file is read
func isTemplate(filename string) bool {
return stringsutil.EqualFoldAny(filepath.Ext(filename), templates.TemplateExtension)
return stringsutil.EqualFoldAny(filepath.Ext(filename), config.GetSupportTemplateFileExtensions()...)
}
// SaveResumeConfig to file

35
v2/pkg/catalog/config/template.go Normal file
View File

@ -0,0 +1,35 @@
package config
import (
"path/filepath"
"strings"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions"
)
// TemplateFormat
type TemplateFormat uint8
const (
YAML TemplateFormat = iota
JSON
Unknown
)
// GetTemplateFormatFromExt returns template format
func GetTemplateFormatFromExt(filePath string) TemplateFormat {
fileExt := strings.ToLower(filepath.Ext(filePath))
switch fileExt {
case extensions.JSON:
return JSON
case extensions.YAML:
return YAML
default:
return Unknown
}
}
// GetSupportedTemplateFileExtensions returns all supported template file extensions
func GetSupportTemplateFileExtensions() []string {
return []string{extensions.YAML, extensions.JSON}
}

26
v2/pkg/catalog/disk/find.go
View File

@ -2,11 +2,14 @@ package disk
import (
"io/fs"
"log"
"os"
"path/filepath"
"strings"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
stringsutil "github.com/projectdiscovery/utils/strings"
)
// GetTemplatesPath returns a list of absolute paths for the provided template list.
@ -16,8 +19,14 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri
allTemplates := []string{}
erred := make(map[string]error)
log.Println(definitions)
for _, t := range definitions {
if strings.HasPrefix(t, "http") && (strings.HasSuffix(t, ".yaml") || strings.HasSuffix(t, ".yml")) {
if stringsutil.ContainsAny(t, knownConfigFiles...) {
// TODO: this is a temporary fix to avoid treating these files as templates
// this should be replaced with more appropriate and robust logic
continue
}
if strings.HasPrefix(t, "http") && stringsutil.ContainsAny(t, config.GetSupportTemplateFileExtensions()...) {
if _, ok := processed[t]; !ok {
processed[t] = true
allTemplates = append(allTemplates, t)
@ -35,7 +44,17 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri
}
}
}
return allTemplates, erred
// purge all falsepositivies
filteredTemplates := []string{}
for _, v := range allTemplates {
// TODO: this is a temporary fix to avoid treating these files as templates
// this should be replaced with more appropriate and robust logic
if !stringsutil.ContainsAny(v, knownConfigFiles...) {
filteredTemplates = append(filteredTemplates, v)
}
}
return filteredTemplates, erred
}
// GetTemplatePath parses the specified input template path and returns a compiled
@ -43,7 +62,6 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri
// or folders provided as in.
func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) {
processed := make(map[string]struct{})
absPath, err := c.convertPathToAbsolute(target)
if err != nil {
return nil, errors.Wrapf(err, "could not find template file")
@ -142,7 +160,7 @@ func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]
if err != nil {
return nil
}
if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown {
if _, ok := processed[path]; !ok {
results = append(results, path)
processed[path] = struct{}{}

3
v2/pkg/catalog/disk/known-files.go Normal file
View File

@ -0,0 +1,3 @@
package disk
var knownConfigFiles = []string{"cves.json", "contributors.json", "TEMPLATES-STATS.json"}

4
v2/pkg/catalog/loader/remote_loader.go
View File

@ -8,8 +8,10 @@ import (
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions"
"github.com/projectdiscovery/nuclei/v2/pkg/utils"
"github.com/projectdiscovery/retryablehttp-go"
stringsutil "github.com/projectdiscovery/utils/strings"
)
type ContentType string
@ -65,7 +67,7 @@ func getRemoteContent(URL string, remoteTemplateDomainList []string, remoteConte
}
return
}
if strings.HasPrefix(URL, "http") && (strings.HasSuffix(URL, ".yaml") || strings.HasSuffix(URL, ".yml")) {
if strings.HasPrefix(URL, "http") && stringsutil.HasSuffixAny(URL, extensions.YAML, extensions.YML) {
remoteContentChannel <- RemoteContent{
Content: []string{URL},
Type: contentType,

17
v2/pkg/parsers/parser.go
View File

@ -1,11 +1,13 @@
package parsers
import (
"encoding/json"
"fmt"
"regexp"
"strings"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/cache"
@ -130,10 +132,17 @@ func ParseTemplate(templatePath string, catalog catalog.Catalog) (*templates.Tem
template.Verified, _ = signer.Verify(signer.DefaultVerifier, data)
}
if NoStrictSyntax {
err = yaml.Unmarshal(data, template)
} else {
err = yaml.UnmarshalStrict(data, template)
switch config.GetTemplateFormatFromExt(templatePath) {
case config.JSON:
err = json.Unmarshal(data, template)
case config.YAML:
if NoStrictSyntax {
err = yaml.Unmarshal(data, template)
} else {
err = yaml.UnmarshalStrict(data, template)
}
default:
err = fmt.Errorf("failed to identify template format expected JSON or YAML but got %v", templatePath)
}
if err != nil {
stats.Increment(SyntaxErrorStats)

7
v2/pkg/templates/extensions/extensions.go Normal file
View File

@ -0,0 +1,7 @@
package extensions
const (
JSON = ".json"
YAML = ".yaml"
YML = ".yml"
)

5
v2/pkg/templates/templates.go
View File

@ -22,11 +22,6 @@ import (
"gopkg.in/yaml.v2"
)
const (
// TemplateExtension defines the template default file extension
TemplateExtension = ".yaml"
)
// Template is a YAML input file which defines all the requests and
// other metadata for a template.
type Template struct {

3
v2/pkg/utils/utils.go
View File

@ -7,6 +7,7 @@ import (
"strings"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/utils/yaml"
"github.com/projectdiscovery/retryablehttp-go"
fileutil "github.com/projectdiscovery/utils/file"
@ -58,7 +59,7 @@ func ReadFromPathOrURL(templatePath string, catalog catalog.Catalog) (data []byt
}
// pre-process directives only for local files
if fileutil.FileExists(templatePath) {
if fileutil.FileExists(templatePath) && config.GetTemplateFormatFromExt(templatePath) == config.YAML {
data, err = yaml.PreProcess(data)
if err != nil {
return nil, err

3
v2/pkg/utils/yaml/preprocess.go
View File

@ -7,6 +7,7 @@ import (
"regexp"
"strings"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions"
fileutil "github.com/projectdiscovery/utils/file"
stringsutil "github.com/projectdiscovery/utils/strings"
)
@ -52,7 +53,7 @@ func PreProcess(data []byte) ([]byte, error) {
return nil, err
}
// if it's yaml, tries to preprocess that too recursively
if stringsutil.HasSuffixAny(includeFileName, ".yaml") {
if stringsutil.HasSuffixAny(includeFileName, extensions.YAML) {
if subIncludedFileContent, err := PreProcess(includeFileContent); err == nil {
includeFileContent = subIncludedFileContent
} else {