mirror of https://github.com/daffainfo/nuclei.git
Add utility to write max-requests to templates (#3607)
* Add utility to write max-requests to templates * fix lint error * fix max-request update edge case * fix convert max-request: 1 => max-request: 1 * WIP, most of the code is commented * Refactor the find and replace logic * Skip if template has the max-requests, do not overwrite - return errors - add warnings * Fix the wrong index calculation - Refactor the getInfoBlock function to not compile regex everytime * Update -tc flag to filter fields within the classification section (#3606) * Add fields from Classification section in a template to the -tc flag expression evaluation Signed-off-by: iamargus95 <kamathsuraj95@gmail.com> * Add tests for filtering Classification section using -tc flag Signed-off-by: iamargus95 <kamathsuraj95@gmail.com> * Fix hyphenated Metadata keys beings added to parameters Signed-off-by: iamargus95 <kamathsuraj95@gmail.com> * Add tests to the fix for hyphenated fields encountered in Metadata section Signed-off-by: iamargus95 <kamathsuraj95@gmail.com> --------- Signed-off-by: iamargus95 <kamathsuraj95@gmail.com> * Check severity att while validating (#3540) * Make severity attribute required * Update test err msg * minor * Do not strict check serverity * Fix failing test * Don't print warning in workflow loader - workflow loader that contains tags load all the template and parse it - i.e it iw printing warning recursively, ignore as the templates already getting valiated * Fix error typo * Resolve comments - split the function into two diff --------- Co-authored-by: Mzack9999 <mzack9999@protonmail.com> Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io> Co-authored-by: shubhamrasal <shubhamdharmarasal@gmail.com> * tlsx dep update (#3620) * updated interactsh version (#3621) * updated interactsh version * workflow update * aws signer: fix missing x-content-sha256 header (#3601) * fix missing x-content-sha256 header * fix variable priority in self-contained templates * remove debug statement * adds generic raw request parser for self-contained req * more integration tests * bug fix: 10x faster race requests * fix failing integration test * chore(deps): bump github.com/xanzy/go-gitlab in /v2 (#3624) Bumps [github.com/xanzy/go-gitlab](https://github.com/xanzy/go-gitlab) from 0.82.0 to 0.83.0. - [Release notes](https://github.com/xanzy/go-gitlab/releases) - [Changelog](https://github.com/xanzy/go-gitlab/blob/master/releases_test.go) - [Commits](https://github.com/xanzy/go-gitlab/compare/v0.82.0...v0.83.0) --- updated-dependencies: - dependency-name: github.com/xanzy/go-gitlab dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/miekg/dns from 1.1.53 to 1.1.54 in /v2 (#3625) Bumps [github.com/miekg/dns](https://github.com/miekg/dns) from 1.1.53 to 1.1.54. - [Release notes](https://github.com/miekg/dns/releases) - [Changelog](https://github.com/miekg/dns/blob/master/Makefile.release) - [Commits](https://github.com/miekg/dns/compare/v1.1.53...v1.1.54) --- updated-dependencies: - dependency-name: github.com/miekg/dns dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /v2 (#3626) Bumps [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) from 1.11.61 to 1.11.64. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.11.61...feature/s3/manager/v1.11.64) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Fix check for OS made in MustDisableSandbox() (#3631) Signed-off-by: iamargus95 <kamathsuraj95@gmail.com> * Fix wrong template loading in dev branch (#3629) * Templates wrong loading * Add tests to cover following scenarios - check optional fields only if template loaded - it should return warning only if template is loaded * enable color in windows (#3634) * enable color in windows * fixed win workflow * typo update * tlsx dep update (#3633) * tlsx dep update * upgrde httpx => 1.3.0 * Fix check for OS made in MustDisableSandbox() (#3631) Signed-off-by: iamargus95 <kamathsuraj95@gmail.com> * Fix wrong template loading in dev branch (#3629) * Templates wrong loading * Add tests to cover following scenarios - check optional fields only if template loaded - it should return warning only if template is loaded * enable color in windows (#3634) * enable color in windows * fixed win workflow * typo update --------- Signed-off-by: iamargus95 <kamathsuraj95@gmail.com> Co-authored-by: Ramana Reddy <ramanaredy.manda@gmail.com> Co-authored-by: Suraj Kamath <kamathsuraj95@gmail.com> Co-authored-by: Shubham Rasal <shubham@projectdiscovery.io> * Expose DNS fields for matchers and extractors (#3613) * Extend dns extractor to dns answer records * add test template * Ignore error for dns variables are not found * Add all the records of answer section * Fixed the wrong typecasting * Issue 3564 var override (#3599) * Check if the variables are override by other means - you can override the template variable value using command line flags * Update lazy eval logic - previously, we were checking any function/expression in variable - now, update the logic, lazy eval only if variable contains any protocol variable(global) * add integration tests * Add test to check the dsl function working in variable * gather all generate variables logic in utils * go mod update * Refactor the generate variables function * go mod update+ fix typo --------- Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io> Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io> * update rod to v0.112.9 #3552 (#3637) * update rod to v0.112.9 * removed unused reflection --------- Co-authored-by: Mzack9999 <mzack9999@protonmail.com> Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io> Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> * change max-requests label to max-request --------- Signed-off-by: iamargus95 <kamathsuraj95@gmail.com> Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: shubhamrasal <shubhamdharmarasal@gmail.com> Co-authored-by: Suraj Kamath <kamathsuraj95@gmail.com> Co-authored-by: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Co-authored-by: Mzack9999 <mzack9999@protonmail.com> Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io> Co-authored-by: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shubham Rasal <shubham@projectdiscovery.io> Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io> Co-authored-by: lu4nx <lx@shellcodes.org>dev
Ramana Reddy
GitHub
parent
ced8d96df2
commit
59376180b1
|
|
@ -10,12 +10,23 @@ import (
|
|||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/gologger/levels"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
"github.com/projectdiscovery/nvd"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
sliceutil "github.com/projectdiscovery/utils/slice"
|
||||
|
|
@ -29,26 +40,64 @@ const (
|
|||
|
||||
var cisaKnownExploitedVulnerabilities map[string]struct{}
|
||||
|
||||
// allTagsRegex is a list of all tags in nuclei templates except id, info, and -
|
||||
var allTagsRegex []*regexp.Regexp
|
||||
var defaultOpts = types.DefaultOptions()
|
||||
|
||||
func init() {
|
||||
var tm templates.Template
|
||||
t := reflect.TypeOf(tm)
|
||||
for i := 0; i < t.NumField(); i++ {
|
||||
tag := t.Field(i).Tag.Get("yaml")
|
||||
if strings.Contains(tag, ",") {
|
||||
tag = strings.Split(tag, ",")[0]
|
||||
}
|
||||
// ignore these tags
|
||||
if tag == "id" || tag == "info" || tag == "" || tag == "-" {
|
||||
continue
|
||||
}
|
||||
re := regexp.MustCompile(tag + `:\s*\n`)
|
||||
allTagsRegex = append(allTagsRegex, re)
|
||||
}
|
||||
|
||||
defaultOpts := types.DefaultOptions()
|
||||
// need to set headless to true for headless templates
|
||||
defaultOpts.Headless = true
|
||||
if err := protocolstate.Init(defaultOpts); err != nil {
|
||||
gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err)
|
||||
}
|
||||
if err := protocolinit.Init(defaultOpts); err != nil {
|
||||
gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err)
|
||||
}
|
||||
if err := fetchCISAKnownExploitedVulnerabilities(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
input = flag.String("i", "", "Templates to annotate")
|
||||
templateDir = flag.String("d", "", "Custom template directory for update")
|
||||
input = flag.String("i", "", "Templates to annotate")
|
||||
verbose = flag.Bool("v", false, "show verbose output")
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
if *input == "" || *templateDir == "" {
|
||||
if *input == "" {
|
||||
log.Fatalf("invalid input, see -h\n")
|
||||
}
|
||||
|
||||
if strings.HasPrefix(*input, "~/") {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to read UserHomeDir: %v, provide absolute template path/directory\n", err)
|
||||
}
|
||||
*input = filepath.Join(home, (*input)[2:])
|
||||
}
|
||||
gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
|
||||
if *verbose {
|
||||
gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
|
||||
}
|
||||
if err := process(); err != nil {
|
||||
log.Fatalf("could not process: %s\n", err)
|
||||
gologger.Error().Msgf("could not process: %s\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -60,8 +109,7 @@ func process() error {
|
|||
defer os.RemoveAll(tempDir)
|
||||
|
||||
client := nvd.NewClientV2()
|
||||
catalog := disk.NewCatalog(*templateDir)
|
||||
|
||||
catalog := disk.NewCatalog(filepath.Dir(*input))
|
||||
paths, err := catalog.GetTemplatePath(*input)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -72,14 +120,18 @@ func process() error {
|
|||
return err
|
||||
}
|
||||
dataString := string(data)
|
||||
|
||||
// First try to resolve references to tags
|
||||
// try to fill max-requests
|
||||
dataString, err = parseAndAddMaxRequests(catalog, path, dataString)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Could not compile max request %s: %s\n", path, err)
|
||||
}
|
||||
// try to resolve references to tags
|
||||
dataString, err = parseAndAddReferenceBasedTags(path, dataString)
|
||||
if err != nil {
|
||||
log.Printf("Could not parse reference tags %s: %s\n", path, err)
|
||||
gologger.Error().Msgf("Could not parse reference tags %s: %s\n", path, err)
|
||||
continue
|
||||
}
|
||||
// Next try and fill CVE data
|
||||
// try and fill CVE data
|
||||
getCVEData(client, path, dataString)
|
||||
}
|
||||
return nil
|
||||
|
|
@ -126,7 +178,7 @@ func getCVEData(client *nvd.ClientV2, filePath, data string) {
|
|||
}
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("Could not parse cisa data %s: %s\n", cveName, err)
|
||||
gologger.Error().Msgf("Could not parse cisa data %s: %s\n", cveName, err)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -138,7 +190,7 @@ func getCVEData(client *nvd.ClientV2, filePath, data string) {
|
|||
|
||||
cveItem, err := client.FetchCVE(cveName)
|
||||
if err != nil {
|
||||
log.Printf("Could not fetch cve %s: %s\n", cveName, err)
|
||||
gologger.Error().Msgf("Could not fetch cve %s: %s\n", cveName, err)
|
||||
return
|
||||
}
|
||||
var cweID []string
|
||||
|
|
@ -149,7 +201,7 @@ func getCVEData(client *nvd.ClientV2, filePath, data string) {
|
|||
}
|
||||
cvssData, err := getPrimaryCVSSData(cveItem)
|
||||
if err != nil {
|
||||
log.Printf("Could not get CVSS data %s: %s\n", cveName, err)
|
||||
gologger.Error().Msgf("Could not get CVSS data %s: %s\n", cveName, err)
|
||||
return
|
||||
}
|
||||
cvssScore := cvssData.BaseScore
|
||||
|
|
@ -175,14 +227,14 @@ func getCVEData(client *nvd.ClientV2, filePath, data string) {
|
|||
infoBlock := InfoBlock{}
|
||||
err = yaml.Unmarshal([]byte(data), &infoBlock)
|
||||
if err != nil {
|
||||
log.Printf("Could not unmarshal info block: %s\n", err)
|
||||
gologger.Warning().Msgf("Could not unmarshal info block: %s\n", err)
|
||||
}
|
||||
|
||||
var changed bool
|
||||
if newSeverity := isSeverityMatchingCvssScore(severityValue, cvssScore); newSeverity != "" {
|
||||
changed = true
|
||||
infoBlock.Info.Severity = newSeverity
|
||||
fmt.Printf("Adjusting severity for %s from %s=>%s (%.2f)\n", filePath, severityValue, newSeverity, cvssScore)
|
||||
gologger.Info().Msgf("Adjusting severity for %s from %s=>%s (%.2f)\n", filePath, severityValue, newSeverity, cvssScore)
|
||||
}
|
||||
isCvssEmpty := cvssScore == 0 || cvssMetrics == ""
|
||||
hasCvssChanged := infoBlock.Info.Classification.CvssScore != cvssScore || cvssMetrics != infoBlock.Info.Classification.CvssMetrics
|
||||
|
|
@ -267,7 +319,7 @@ func getCVEData(client *nvd.ClientV2, filePath, data string) {
|
|||
yamlEncoder.SetIndent(yamlIndentSpaces)
|
||||
err = yamlEncoder.Encode(infoBlock)
|
||||
if err != nil {
|
||||
log.Printf("Could not marshal info block: %s\n", err)
|
||||
gologger.Warning().Msgf("Could not marshal info block: %s\n", err)
|
||||
return
|
||||
}
|
||||
newInfoBlockData := strings.TrimSuffix(newInfoBlock.String(), "\n")
|
||||
|
|
@ -275,7 +327,7 @@ func getCVEData(client *nvd.ClientV2, filePath, data string) {
|
|||
newTemplate := strings.ReplaceAll(data, infoBlockClean, newInfoBlockData)
|
||||
if changed {
|
||||
_ = os.WriteFile(filePath, []byte(newTemplate), 0644)
|
||||
fmt.Printf("Wrote updated template to %s\n", filePath)
|
||||
gologger.Info().Msgf("Wrote updated template to %s\n", filePath)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -503,11 +555,96 @@ type TemplateClassification struct {
|
|||
type TemplateInfo struct {
|
||||
Name string `yaml:"name"`
|
||||
Author string `yaml:"author"`
|
||||
Severity string `yaml:"severity"`
|
||||
Severity string `yaml:"severity,omitempty"`
|
||||
Description string `yaml:"description,omitempty"`
|
||||
Reference []string `yaml:"reference,omitempty"`
|
||||
Remediation string `yaml:"remediation,omitempty"`
|
||||
Classification TemplateClassification `yaml:"classification,omitempty"`
|
||||
Metadata map[string]string `yaml:"metadata,omitempty"`
|
||||
Metadata map[string]interface{} `yaml:"metadata,omitempty"`
|
||||
Tags string `yaml:"tags,omitempty"`
|
||||
}
|
||||
|
||||
// parseAndAddMaxRequests parses and adds max requests to templates
|
||||
func parseAndAddMaxRequests(catalog catalog.Catalog, path, data string) (string, error) {
|
||||
template, err := parseTemplate(catalog, path)
|
||||
if err != nil {
|
||||
gologger.Warning().Label("max-request").Msgf("Could not parse template: %s\n", err)
|
||||
return data, err
|
||||
}
|
||||
|
||||
if template.TotalRequests < 1 {
|
||||
return data, nil
|
||||
}
|
||||
// Marshal the updated info block back to YAML.
|
||||
infoBlockStart, infoBlockEnd := getInfoStartEnd(data)
|
||||
infoBlockOrig := data[infoBlockStart:infoBlockEnd]
|
||||
infoBlockOrig = strings.TrimRight(infoBlockOrig, "\n")
|
||||
|
||||
infoBlock := InfoBlock{}
|
||||
err = yaml.Unmarshal([]byte(data), &infoBlock)
|
||||
if err != nil {
|
||||
gologger.Warning().Label("max-request").Msgf("Could not unmarshal info block: %s\n", err)
|
||||
return data, err
|
||||
}
|
||||
// if metadata is nil, create a new map
|
||||
if infoBlock.Info.Metadata == nil {
|
||||
infoBlock.Info.Metadata = make(map[string]interface{})
|
||||
}
|
||||
// do not update if it is already present and equal
|
||||
if mr, ok := infoBlock.Info.Metadata["max-request"]; ok && mr.(int) == template.TotalRequests {
|
||||
return data, nil
|
||||
}
|
||||
infoBlock.Info.Metadata["max-request"] = template.TotalRequests
|
||||
|
||||
var newInfoBlock bytes.Buffer
|
||||
yamlEncoder := yaml.NewEncoder(&newInfoBlock)
|
||||
yamlEncoder.SetIndent(yamlIndentSpaces)
|
||||
err = yamlEncoder.Encode(infoBlock)
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("Could not marshal info block: %s\n", err)
|
||||
return data, err
|
||||
}
|
||||
newInfoBlockData := strings.TrimSuffix(newInfoBlock.String(), "\n")
|
||||
|
||||
// replace old info block with new info block
|
||||
newTemplate := strings.ReplaceAll(data, infoBlockOrig, newInfoBlockData)
|
||||
|
||||
err = os.WriteFile(path, []byte(newTemplate), 0644)
|
||||
if err == nil {
|
||||
gologger.Info().Label("max-request").Msgf("Wrote updated template to %s\n", path)
|
||||
}
|
||||
return newTemplate, err
|
||||
}
|
||||
|
||||
// parseTemplate parses a template and returns the template object
|
||||
func parseTemplate(catalog catalog.Catalog, templatePath string) (*templates.Template, error) {
|
||||
executerOpts := protocols.ExecuterOptions{
|
||||
Catalog: catalog,
|
||||
Options: defaultOpts,
|
||||
}
|
||||
reader, err := executerOpts.Catalog.OpenFile(templatePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
template, err := templates.ParseTemplateFromReader(reader, nil, executerOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return template, nil
|
||||
}
|
||||
|
||||
// find the start and end of the info block
|
||||
func getInfoStartEnd(data string) (int, int) {
|
||||
info := strings.Index(data, "info:")
|
||||
var indices []int
|
||||
for _, re := range allTagsRegex {
|
||||
// find the first occurance of the label
|
||||
match := re.FindStringIndex(data)
|
||||
if match != nil {
|
||||
indices = append(indices, match[0])
|
||||
}
|
||||
}
|
||||
// find the first one after info block
|
||||
sort.Ints(indices)
|
||||
return info, indices[0] - 1
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue