Use templateman enhance api to populate CVE info (#3788)

* use templateman enhance api to populate cve info

* rename cve-annotate => tmc
add additional flags to format, lint and enhance template using templateman apis

* minior changes

* remove duplicate code

* misc update

* Add validate and error log option

* print if updated

* print format and enhance only if updated

* make max-request optional

* fix reference unmarshal error

* fix removing self-contained tag

---------

Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>
dev
Ramana Reddy 2023-06-21 18:49:40 +05:30 committed by GitHub
parent 2a32ed9cba
commit 936256460b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 517 additions and 671 deletions

View File

@ -23,8 +23,8 @@ builds:
flags:
- -trimpath
- main: cmd/cve-annotate/main.go
binary: cve-annotate
- main: cmd/tmc/main.go
binary: tmc
id: annotate
env:

View File

@ -1,650 +0,0 @@
package main
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"path/filepath"
"reflect"
"regexp"
"sort"
"strconv"
"strings"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gologger/levels"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/nvd"
"github.com/projectdiscovery/retryablehttp-go"
sliceutil "github.com/projectdiscovery/utils/slice"
stringsutil "github.com/projectdiscovery/utils/strings"
"gopkg.in/yaml.v3"
)
const (
yamlIndentSpaces = 2
)
var cisaKnownExploitedVulnerabilities map[string]struct{}
// allTagsRegex is a list of all tags in nuclei templates except id, info, and -
var allTagsRegex []*regexp.Regexp
var defaultOpts = types.DefaultOptions()
func init() {
var tm templates.Template
t := reflect.TypeOf(tm)
for i := 0; i < t.NumField(); i++ {
tag := t.Field(i).Tag.Get("yaml")
if strings.Contains(tag, ",") {
tag = strings.Split(tag, ",")[0]
}
// ignore these tags
if tag == "id" || tag == "info" || tag == "" || tag == "-" {
continue
}
re := regexp.MustCompile(tag + `:\s*\n`)
allTagsRegex = append(allTagsRegex, re)
}
defaultOpts := types.DefaultOptions()
// need to set headless to true for headless templates
defaultOpts.Headless = true
if err := protocolstate.Init(defaultOpts); err != nil {
gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err)
}
if err := protocolinit.Init(defaultOpts); err != nil {
gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err)
}
if err := fetchCISAKnownExploitedVulnerabilities(); err != nil {
panic(err)
}
}
var (
input = flag.String("i", "", "Templates to annotate")
verbose = flag.Bool("v", false, "show verbose output")
)
func main() {
flag.Parse()
if *input == "" {
log.Fatalf("invalid input, see -h\n")
}
if strings.HasPrefix(*input, "~/") {
home, err := os.UserHomeDir()
if err != nil {
log.Fatalf("Failed to read UserHomeDir: %v, provide absolute template path/directory\n", err)
}
*input = filepath.Join(home, (*input)[2:])
}
gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
if *verbose {
gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
}
if err := process(); err != nil {
gologger.Error().Msgf("could not process: %s\n", err)
}
}
func process() error {
tempDir, err := os.MkdirTemp("", "nuclei-nvd-%s")
if err != nil {
return err
}
defer os.RemoveAll(tempDir)
client := nvd.NewClientV2()
templateCatalog := disk.NewCatalog(filepath.Dir(*input))
paths, err := templateCatalog.GetTemplatePath(*input)
if err != nil {
return err
}
for _, path := range paths {
data, err := os.ReadFile(path)
if err != nil {
return err
}
dataString := string(data)
// try to fill max-requests
dataString, err = parseAndAddMaxRequests(templateCatalog, path, dataString)
if err != nil {
gologger.Error().Msgf("Could not compile max request %s: %s\n", path, err)
}
// try to resolve references to tags
dataString, err = parseAndAddReferenceBasedTags(path, dataString)
if err != nil {
gologger.Error().Msgf("Could not parse reference tags %s: %s\n", path, err)
continue
}
// try and fill CVE data
getCVEData(client, path, dataString)
}
return nil
}
var (
idRegex = regexp.MustCompile("id: ([C|c][V|v][E|e]-[0-9]+-[0-9]+)")
severityRegex = regexp.MustCompile(`severity: ([a-z]+)`)
)
const maxReferenceCount = 5
// dead sites to skip for references
var badRefs = []string{
"osvdb.org/",
"securityfocus.com/",
"archives.neohapsis.com/",
"iss.net/",
"ntelbras.com/",
"andmp.com/",
"blacklanternsecurity.com/",
"pwnwiki.org/",
"0dayhack.net/",
"correkt.horse/",
"poc.wgpsec.org/",
"ctf-writeup.revers3c.com/",
"secunia.com/",
}
func getCVEData(client *nvd.ClientV2, filePath, data string) {
matches := idRegex.FindAllStringSubmatch(data, 1)
if len(matches) == 0 {
return
}
cveName := matches[0][1]
// Perform CISA Known-exploited-vulnerabilities tag annotation
// if we discover it has been exploited.
var err error
if cisaKnownExploitedVulnerabilities != nil {
_, ok := cisaKnownExploitedVulnerabilities[strings.ToLower(cveName)]
if ok {
data, err = parseAndAddCISAKevTagTemplate(filePath, data)
}
}
if err != nil {
gologger.Error().Msgf("Could not parse cisa data %s: %s\n", cveName, err)
return
}
severityMatches := severityRegex.FindAllStringSubmatch(data, 1)
if len(severityMatches) == 0 {
return
}
severityValue := severityMatches[0][1]
cveItem, err := client.FetchCVE(cveName)
if err != nil {
gologger.Error().Msgf("Could not fetch cve %s: %s\n", cveName, err)
return
}
var cweID []string
for _, weaknessData := range cveItem.Cve.Weaknesses {
for _, description := range weaknessData.Description {
cweID = append(cweID, description.Value)
}
}
cvssData, err := getPrimaryCVSSData(cveItem)
if err != nil {
gologger.Error().Msgf("Could not get CVSS data %s: %s\n", cveName, err)
return
}
cvssScore := cvssData.BaseScore
cvssMetrics := cvssData.VectorString
// Perform some hacky string replacement to place the metadata in templates
infoBlockIndexData := data[strings.Index(data, "info:"):]
requestsIndex := strings.Index(infoBlockIndexData, "requests:")
networkIndex := strings.Index(infoBlockIndexData, "network:")
variablesIndex := strings.Index(infoBlockIndexData, "variables:")
if requestsIndex == -1 && networkIndex == -1 && variablesIndex == -1 {
return
}
if networkIndex != -1 {
requestsIndex = networkIndex
}
if variablesIndex != -1 {
requestsIndex = variablesIndex
}
infoBlockData := infoBlockIndexData[:requestsIndex]
infoBlockClean := strings.TrimRight(infoBlockData, "\n")
infoBlock := InfoBlock{}
err = yaml.Unmarshal([]byte(data), &infoBlock)
if err != nil {
gologger.Warning().Msgf("Could not unmarshal info block: %s\n", err)
}
var changed bool
if newSeverity := isSeverityMatchingCvssScore(severityValue, cvssScore); newSeverity != "" {
changed = true
infoBlock.Info.Severity = newSeverity
gologger.Info().Msgf("Adjusting severity for %s from %s=>%s (%.2f)\n", filePath, severityValue, newSeverity, cvssScore)
}
isCvssEmpty := cvssScore == 0 || cvssMetrics == ""
hasCvssChanged := infoBlock.Info.Classification.CvssScore != cvssScore || cvssMetrics != infoBlock.Info.Classification.CvssMetrics
if !isCvssEmpty && hasCvssChanged {
changed = true
infoBlock.Info.Classification.CvssMetrics = cvssMetrics
infoBlock.Info.Classification.CvssScore = cvssScore
infoBlock.Info.Classification.CveId = cveName
if len(cweID) > 0 && (cweID[0] != "NVD-CWE-Other" && cweID[0] != "NVD-CWE-noinfo") {
infoBlock.Info.Classification.CweId = strings.Join(cweID, ",")
}
}
// If there is no description field, fill the description from CVE information
enDescription, err := getEnglishLangString(cveItem.Cve.Descriptions)
hasDescriptionData := err != nil
isDescriptionEmpty := infoBlock.Info.Description == ""
if isDescriptionEmpty && hasDescriptionData {
changed = true
// removes all new lines
description := stringsutil.ReplaceAll(enDescription, "", "\n", "\\", "'", "\t")
description += "\n"
infoBlock.Info.Description = description
}
// we are unmarshaling info block to have valid data
var referenceDataURLs []string
// skip sites that are no longer alive
for _, reference := range cveItem.Cve.References {
if stringsutil.ContainsAny(reference.URL, badRefs...) {
continue
}
referenceDataURLs = append(referenceDataURLs, reference.URL)
}
hasReferenceData := len(cveItem.Cve.References) > 0
areCveReferencesContained := sliceutil.ContainsItems(infoBlock.Info.Reference, referenceDataURLs)
referencesCount := len(infoBlock.Info.Reference)
if hasReferenceData && !areCveReferencesContained {
changed = true
for _, ref := range referenceDataURLs {
referencesCount++
if referencesCount >= maxReferenceCount {
break
}
infoBlock.Info.Reference = append(infoBlock.Info.Reference, ref)
}
infoBlock.Info.Reference = sliceutil.PruneEmptyStrings(sliceutil.Dedupe(infoBlock.Info.Reference))
}
cpeSet := map[string]bool{}
for _, config := range cveItem.Cve.Configurations {
// Right now this covers only simple configurations. More complex configurations can have multiple CPEs
if len(config.Nodes) == 1 {
changed = true
node := config.Nodes[0]
for _, match := range node.CpeMatch {
cpeSet[extractVersionlessCpe(match.Criteria)] = true
}
}
}
uniqueCpes := make([]string, 0, len(cpeSet))
for k := range cpeSet {
uniqueCpes = append(uniqueCpes, k)
}
if len(uniqueCpes) == 1 {
infoBlock.Info.Classification.Cpe = uniqueCpes[0]
}
epss, err := fetchEpss(cveName)
if err != nil {
log.Printf("Could not fetch Epss score: %s\n", err)
return
}
hasEpssChanged := epss != infoBlock.Info.Classification.EpssScore
if hasEpssChanged {
changed = true
infoBlock.Info.Classification.EpssScore = epss
}
var newInfoBlock bytes.Buffer
yamlEncoder := yaml.NewEncoder(&newInfoBlock)
yamlEncoder.SetIndent(yamlIndentSpaces)
err = yamlEncoder.Encode(infoBlock)
if err != nil {
gologger.Warning().Msgf("Could not marshal info block: %s\n", err)
return
}
newInfoBlockData := strings.TrimSuffix(newInfoBlock.String(), "\n")
newTemplate := strings.ReplaceAll(data, infoBlockClean, newInfoBlockData)
if changed {
_ = os.WriteFile(filePath, []byte(newTemplate), 0644)
gologger.Info().Msgf("Wrote updated template to %s\n", filePath)
}
}
func getPrimaryCVSSData(vuln nvd.Vulnerability) (nvd.CvssData, error) {
for _, data := range vuln.Cve.Metrics.CvssMetricV31 {
if data.Type == "Primary" {
return data.CvssData, nil
}
}
for _, data := range vuln.Cve.Metrics.CvssMetricV3 {
if data.Type == "Primary" {
return data.CvssData, nil
}
}
return nvd.CvssData{}, fmt.Errorf("no primary cvss metric found")
}
func getEnglishLangString(data []nvd.LangString) (string, error) {
for _, item := range data {
if item.Lang == "en" {
return item.Value, nil
}
}
return "", fmt.Errorf("no english item found")
}
func isSeverityMatchingCvssScore(severity string, score float64) string {
if score == 0.0 {
return ""
}
var expected string
if score >= 0.1 && score <= 3.9 {
expected = "low"
} else if score >= 4.0 && score <= 6.9 {
expected = "medium"
} else if score >= 7.0 && score <= 8.9 {
expected = "high"
} else if score >= 9.0 && score <= 10.0 {
expected = "critical"
}
if expected != "" && expected != severity {
return expected
}
return ""
}
func extractVersionlessCpe(cpe string) string {
parts := strings.Split(cpe, ":")
versionlessPart := parts[0:5]
rest := strings.Split(strings.Repeat("*", len(parts)-len(versionlessPart)), "")
return strings.Join(append(versionlessPart, rest...), ":")
}
type ApiFirstEpssResponse struct {
Status string `json:"status"`
StatusCode int `json:"status-code"`
Version string `json:"version"`
Access string `json:"access"`
Total int `json:"total"`
Offset int `json:"offset"`
Limit int `json:"limit"`
Data []struct {
Cve string `json:"cve"`
Epss string `json:"epss"`
Percentile string `json:"percentile"`
Date string `json:"date"`
} `json:"data"`
}
func fetchEpss(cveId string) (float64, error) {
resp, err := http.Get(fmt.Sprintf("https://api.first.org/data/v1/epss?cve=%s", cveId))
if err != nil {
return 0, fmt.Errorf("unable to fetch EPSS data from first.org: %v", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return 0, fmt.Errorf("unable to read reponse body: %v", err)
}
var parsedResp ApiFirstEpssResponse
err = json.Unmarshal(body, &parsedResp)
if err != nil {
return 0, fmt.Errorf("error while parsing EPSS response: %v", err)
}
if len(parsedResp.Data) != 1 {
return 0, fmt.Errorf("unexpected number of results in EPSS response. Expecting exactly 1, got %v", len(parsedResp.Data))
}
epss := parsedResp.Data[0].Epss
return strconv.ParseFloat(epss, 64)
}
type cisaKEVData struct {
Vulnerabilities []struct {
CVEID string `json:"cveID"`
}
}
// fetchCISAKnownExploitedVulnerabilities fetches CISA known exploited
// vulnerabilities catalog for template tag enrichment
func fetchCISAKnownExploitedVulnerabilities() error {
data := &cisaKEVData{}
resp, err := retryablehttp.DefaultClient().Get("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json")
if err != nil {
return errors.Wrap(err, "could not get cisa kev catalog")
}
defer resp.Body.Close()
if err := json.NewDecoder(resp.Body).Decode(data); err != nil {
return errors.Wrap(err, "could not decode cisa kev catalog json data")
}
cisaKnownExploitedVulnerabilities = make(map[string]struct{})
for _, vuln := range data.Vulnerabilities {
cisaKnownExploitedVulnerabilities[strings.ToLower(vuln.CVEID)] = struct{}{}
}
return nil
}
// parseAndAddCISAKevTagTemplate parses and adds `kev` tag to CISA KEV templates.
// also removes cisa tag if it exists
func parseAndAddCISAKevTagTemplate(path string, data string) (string, error) {
block := &InfoBlock{}
if err := yaml.NewDecoder(strings.NewReader(data)).Decode(block); err != nil {
return "", errors.Wrap(err, "could not decode template yaml")
}
splitted := strings.Split(block.Info.Tags, ",")
if len(splitted) == 0 {
return data, nil
}
var cisaIndex = -1
for i, tag := range splitted {
// If we already have tag, return
if tag == "kev" {
return data, nil
}
if tag == "cisa" {
cisaIndex = i
}
}
// Remove CISA index tag element
if cisaIndex >= 0 {
splitted = append(splitted[:cisaIndex], splitted[cisaIndex+1:]...)
}
splitted = append(splitted, "kev")
replaced := strings.ReplaceAll(data, block.Info.Tags, strings.Join(splitted, ","))
return replaced, os.WriteFile(path, []byte(replaced), os.ModePerm)
}
// parseAndAddReferenceBasedTags parses and adds reference based tags to templates
func parseAndAddReferenceBasedTags(path string, data string) (string, error) {
block := &InfoBlock{}
if err := yaml.NewDecoder(strings.NewReader(data)).Decode(block); err != nil {
return "", errors.Wrap(err, "could not decode template yaml")
}
splitted := strings.Split(block.Info.Tags, ",")
if len(splitted) == 0 {
return data, nil
}
tagsCurrent := fmt.Sprintf("tags: %s", block.Info.Tags)
newTags := suggestTagsBasedOnReference(block.Info.Reference, splitted)
if len(newTags) == len(splitted) {
return data, nil
}
replaced := strings.ReplaceAll(data, tagsCurrent, fmt.Sprintf("tags: %s", strings.Join(newTags, ",")))
return replaced, os.WriteFile(path, []byte(replaced), os.ModePerm)
}
var referenceMapping = map[string]string{
"huntr.dev": "huntr",
"hackerone.com": "hackerone",
"tenable.com": "tenable",
"packetstormsecurity.org": "packetstorm",
"seclists.org": "seclists",
"wpscan.com": "wpscan",
"packetstormsecurity.com": "packetstorm",
"exploit-db.com": "edb",
"https://github.com/rapid7/metasploit-framework/": "msf",
"https://github.com/vulhub/vulhub/": "vulhub",
}
func suggestTagsBasedOnReference(references, currentTags []string) []string {
uniqueTags := make(map[string]struct{})
for _, value := range currentTags {
uniqueTags[value] = struct{}{}
}
for _, reference := range references {
parsed, err := url.Parse(reference)
if err != nil {
continue
}
hostname := parsed.Hostname()
for value, tag := range referenceMapping {
if strings.HasSuffix(hostname, value) || strings.HasPrefix(reference, value) {
uniqueTags[tag] = struct{}{}
}
}
}
newTags := make([]string, 0, len(uniqueTags))
for tag := range uniqueTags {
newTags = append(newTags, tag)
}
return newTags
}
// InfoBlock Cloning struct from nuclei as we don't want any validation
type InfoBlock struct {
Info TemplateInfo `yaml:"info"`
}
type TemplateClassification struct {
CvssMetrics string `yaml:"cvss-metrics,omitempty"`
CvssScore float64 `yaml:"cvss-score,omitempty"`
CveId string `yaml:"cve-id,omitempty"`
CweId string `yaml:"cwe-id,omitempty"`
Cpe string `yaml:"cpe,omitempty"`
EpssScore float64 `yaml:"epss-score,omitempty"`
}
type TemplateInfo struct {
Name string `yaml:"name"`
Author string `yaml:"author"`
Severity string `yaml:"severity,omitempty"`
Description string `yaml:"description,omitempty"`
Reference []string `yaml:"reference,omitempty"`
Remediation string `yaml:"remediation,omitempty"`
Classification TemplateClassification `yaml:"classification,omitempty"`
Metadata map[string]interface{} `yaml:"metadata,omitempty"`
Tags string `yaml:"tags,omitempty"`
}
// parseAndAddMaxRequests parses and adds max requests to templates
func parseAndAddMaxRequests(catalog catalog.Catalog, path, data string) (string, error) {
template, err := parseTemplate(catalog, path)
if err != nil {
gologger.Warning().Label("max-request").Msgf("Could not parse template: %s\n", err)
return data, err
}
if template.TotalRequests < 1 {
return data, nil
}
// Marshal the updated info block back to YAML.
infoBlockStart, infoBlockEnd := getInfoStartEnd(data)
infoBlockOrig := data[infoBlockStart:infoBlockEnd]
infoBlockOrig = strings.TrimRight(infoBlockOrig, "\n")
infoBlock := InfoBlock{}
err = yaml.Unmarshal([]byte(data), &infoBlock)
if err != nil {
gologger.Warning().Label("max-request").Msgf("Could not unmarshal info block: %s\n", err)
return data, err
}
// if metadata is nil, create a new map
if infoBlock.Info.Metadata == nil {
infoBlock.Info.Metadata = make(map[string]interface{})
}
// do not update if it is already present and equal
if mr, ok := infoBlock.Info.Metadata["max-request"]; ok && mr.(int) == template.TotalRequests {
return data, nil
}
infoBlock.Info.Metadata["max-request"] = template.TotalRequests
var newInfoBlock bytes.Buffer
yamlEncoder := yaml.NewEncoder(&newInfoBlock)
yamlEncoder.SetIndent(yamlIndentSpaces)
err = yamlEncoder.Encode(infoBlock)
if err != nil {
gologger.Warning().Msgf("Could not marshal info block: %s\n", err)
return data, err
}
newInfoBlockData := strings.TrimSuffix(newInfoBlock.String(), "\n")
// replace old info block with new info block
newTemplate := strings.ReplaceAll(data, infoBlockOrig, newInfoBlockData)
err = os.WriteFile(path, []byte(newTemplate), 0644)
if err == nil {
gologger.Info().Label("max-request").Msgf("Wrote updated template to %s\n", path)
}
return newTemplate, err
}
// parseTemplate parses a template and returns the template object
func parseTemplate(catalog catalog.Catalog, templatePath string) (*templates.Template, error) {
executorOpts := protocols.ExecutorOptions{
Catalog: catalog,
Options: defaultOpts,
}
reader, err := executorOpts.Catalog.OpenFile(templatePath)
if err != nil {
return nil, err
}
template, err := templates.ParseTemplateFromReader(reader, nil, executorOpts)
if err != nil {
return nil, err
}
return template, nil
}
// find the start and end of the info block
func getInfoStartEnd(data string) (int, int) {
info := strings.Index(data, "info:")
var indices []int
for _, re := range allTagsRegex {
// find the first occurrence of the label
match := re.FindStringIndex(data)
if match != nil {
indices = append(indices, match[0])
}
}
// find the first one after info block
sort.Ints(indices)
return info, indices[0] - 1
}

435
v2/cmd/tmc/main.go Normal file
View File

@ -0,0 +1,435 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"reflect"
"regexp"
"sort"
"strings"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gologger/levels"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/retryablehttp-go"
errorutil "github.com/projectdiscovery/utils/errors"
"gopkg.in/yaml.v3"
)
const (
yamlIndentSpaces = 2
// templateman api base url
tmBaseUrlDefault = "https://tm.nuclei.sh"
)
var tmBaseUrl string
func init() {
tmBaseUrl = os.Getenv("TEMPLATEMAN_SERVER")
if tmBaseUrl == "" {
tmBaseUrl = tmBaseUrlDefault
}
}
// allTagsRegex is a list of all tags in nuclei templates except id, info, and -
var allTagsRegex []*regexp.Regexp
var defaultOpts = types.DefaultOptions()
func init() {
var tm templates.Template
t := reflect.TypeOf(tm)
for i := 0; i < t.NumField(); i++ {
tag := t.Field(i).Tag.Get("yaml")
if strings.Contains(tag, ",") {
tag = strings.Split(tag, ",")[0]
}
// ignore these tags
if tag == "id" || tag == "info" || tag == "" || tag == "-" {
continue
}
re := regexp.MustCompile(tag + `:\s*\n`)
if t.Field(i).Type.Kind() == reflect.Bool {
re = regexp.MustCompile(tag + `:\s*(true|false)\s*\n`)
}
allTagsRegex = append(allTagsRegex, re)
}
defaultOpts := types.DefaultOptions()
// need to set headless to true for headless templates
defaultOpts.Headless = true
if err := protocolstate.Init(defaultOpts); err != nil {
gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err)
}
if err := protocolinit.Init(defaultOpts); err != nil {
gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err)
}
}
var idRegex = regexp.MustCompile("id: ([C|c][V|v][E|e]-[0-9]+-[0-9]+)")
type options struct {
input string
errorLogFile string
lint bool
validate bool
format bool
enhance bool
maxRequest bool
debug bool
}
func main() {
opts := options{}
flagSet := goflags.NewFlagSet()
flagSet.SetDescription(`TemplateMan CLI is baisc utility built on the TemplateMan API to standardize nuclei templates.`)
flagSet.CreateGroup("Input", "input",
flagSet.StringVarP(&opts.input, "input", "i", "", "Templates to annotate"),
)
flagSet.CreateGroup("Config", "config",
flagSet.BoolVarP(&opts.lint, "lint", "l", false, "lint given nuclei template"),
flagSet.BoolVarP(&opts.validate, "validate", "v", false, "validate given nuclei template"),
flagSet.BoolVarP(&opts.format, "format", "f", false, "format given nuclei template"),
flagSet.BoolVarP(&opts.enhance, "enhance", "e", false, "enhance given nuclei template"),
flagSet.BoolVarP(&opts.maxRequest, "max-request", "mr", false, "add / update max request counter"),
flagSet.StringVarP(&opts.errorLogFile, "error-log", "el", "", "file to write failed template update"),
flagSet.BoolVarP(&opts.debug, "debug", "d", false, "show debug message"),
)
if err := flagSet.Parse(); err != nil {
gologger.Fatal().Msgf("Error parsing flags: %s\n", err)
}
if opts.input == "" {
gologger.Fatal().Msg("input template path/directory is required")
}
if strings.HasPrefix(opts.input, "~/") {
home, err := os.UserHomeDir()
if err != nil {
log.Fatalf("Failed to read UserHomeDir: %v, provide absolute template path/directory\n", err)
}
opts.input = filepath.Join(home, (opts.input)[2:])
}
gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo)
if opts.debug {
gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
}
if err := process(opts); err != nil {
gologger.Error().Msgf("could not process: %s\n", err)
}
}
func process(opts options) error {
tempDir, err := os.MkdirTemp("", "nuclei-nvd-%s")
if err != nil {
return err
}
defer os.RemoveAll(tempDir)
var errFile *os.File
if opts.errorLogFile != "" {
errFile, err = os.OpenFile(opts.errorLogFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
gologger.Fatal().Msgf("could not open error log file: %s\n", err)
}
defer errFile.Close()
}
templateCatalog := disk.NewCatalog(filepath.Dir(opts.input))
paths, err := templateCatalog.GetTemplatePath(opts.input)
if err != nil {
return err
}
for _, path := range paths {
data, err := os.ReadFile(path)
if err != nil {
return err
}
dataString := string(data)
if opts.maxRequest {
var updated bool // if max-requests is updated
dataString, updated, err = parseAndAddMaxRequests(templateCatalog, path, dataString)
if err != nil {
gologger.Info().Label("max-request").Msgf(logErrMsg(path, err, opts.debug, errFile))
} else {
if updated {
gologger.Info().Label("max-request").Msgf("✅ updated template: %s\n", path)
}
// do not print if max-requests is not updated
}
}
if opts.lint {
lint, err := lintTemplate(dataString)
if err != nil {
gologger.Info().Label("lint").Msg(logErrMsg(path, err, opts.debug, errFile))
}
if lint {
gologger.Info().Label("lint").Msgf("✅ lint template: %s\n", path)
}
}
if opts.validate {
validate, err := validateTemplate(dataString)
if err != nil {
gologger.Info().Label("validate").Msg(logErrMsg(path, err, opts.debug, errFile))
}
if validate {
gologger.Info().Label("validate").Msgf("✅ validated template: %s\n", path)
}
}
if opts.format {
formatedTemplateData, isFormated, err := formatTemplate(dataString)
if err != nil {
gologger.Info().Label("format").Msg(logErrMsg(path, err, opts.debug, errFile))
} else {
if isFormated {
_ = os.WriteFile(path, []byte(formatedTemplateData), 0644)
dataString = formatedTemplateData
gologger.Info().Label("format").Msgf("✅ formated template: %s\n", path)
}
}
}
if opts.enhance {
// currently enhance api only supports cve-id's
matches := idRegex.FindAllStringSubmatch(dataString, 1)
if len(matches) == 0 {
continue
}
enhancedTemplateData, isEnhanced, err := enhanceTemplate(dataString)
if err != nil {
gologger.Info().Label("enhance").Msg(logErrMsg(path, err, opts.debug, errFile))
continue
} else {
if isEnhanced {
_ = os.WriteFile(path, []byte(enhancedTemplateData), 0644)
gologger.Info().Label("enhance").Msgf("✅ updated template: %s\n", path)
}
}
}
}
return nil
}
func logErrMsg(path string, err error, debug bool, errFile *os.File) string {
msg := fmt.Sprintf("❌ template: %s\n", path)
if debug {
msg = fmt.Sprintf("❌ template: %s err: %s\n", path, err)
}
if errFile != nil {
_, _ = errFile.WriteString(fmt.Sprintf("❌ template: %s err: %s\n", path, err))
}
return msg
}
// enhanceTemplateData enhances template data using templateman
// ref: https://github.com/projectdiscovery/templateman/blob/main/templateman-rest-api/README.md#enhance-api
func enhanceTemplate(data string) (string, bool, error) {
resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/enhance", tmBaseUrl), "application/x-yaml", strings.NewReader(data))
if err != nil {
return data, false, err
}
if resp.StatusCode != 200 {
return data, false, errorutil.New("unexpected status code: %v", resp.Status)
}
var templateResp TemplateResp
if err := json.NewDecoder(resp.Body).Decode(&templateResp); err != nil {
return data, false, err
}
if strings.TrimSpace(templateResp.Enhanced) != "" {
return templateResp.Enhanced, templateResp.Enhance, nil
}
if templateResp.ValidateErrorCount > 0 {
if len(templateResp.ValidateError) > 0 {
return data, false, errorutil.NewWithTag("validate", templateResp.ValidateError[0].Message+": at line %v", templateResp.ValidateError[0].Mark.Line)
}
return data, false, errorutil.New("validation failed").WithTag("validate")
}
if templateResp.Error.Name != "" {
return data, false, errorutil.New(templateResp.Error.Name)
}
if strings.TrimSpace(templateResp.Enhanced) == "" && !templateResp.Lint {
if templateResp.LintError.Reason != "" {
return data, false, errorutil.NewWithTag("lint", templateResp.LintError.Reason+" : at line %v", templateResp.LintError.Mark.Line)
}
return data, false, errorutil.NewWithTag("lint", "at line: %v", templateResp.LintError.Mark.Line)
}
return data, false, errorutil.New("template enhance failed")
}
// formatTemplateData formats template data using templateman format api
func formatTemplate(data string) (string, bool, error) {
resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/format", tmBaseUrl), "application/x-yaml", strings.NewReader(data))
if err != nil {
return data, false, err
}
if resp.StatusCode != 200 {
return data, false, errorutil.New("unexpected status code: %v", resp.Status)
}
var templateResp TemplateResp
if err := json.NewDecoder(resp.Body).Decode(&templateResp); err != nil {
return data, false, err
}
if strings.TrimSpace(templateResp.Updated) != "" {
return templateResp.Updated, templateResp.Format, nil
}
if templateResp.ValidateErrorCount > 0 {
if len(templateResp.ValidateError) > 0 {
return data, false, errorutil.NewWithTag("validate", templateResp.ValidateError[0].Message+": at line %v", templateResp.ValidateError[0].Mark.Line)
}
return data, false, errorutil.New("validation failed").WithTag("validate")
}
if templateResp.Error.Name != "" {
return data, false, errorutil.New(templateResp.Error.Name)
}
if strings.TrimSpace(templateResp.Updated) == "" && !templateResp.Lint {
if templateResp.LintError.Reason != "" {
return data, false, errorutil.NewWithTag("lint", templateResp.LintError.Reason+" : at line %v", templateResp.LintError.Mark.Line)
}
return data, false, errorutil.NewWithTag("lint", "at line: %v", templateResp.LintError.Mark.Line)
}
return data, false, errorutil.New("template format failed")
}
// lintTemplateData lints template data using templateman lint api
func lintTemplate(data string) (bool, error) {
resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/lint", tmBaseUrl), "application/x-yaml", strings.NewReader(data))
if err != nil {
return false, err
}
if resp.StatusCode != 200 {
return false, errorutil.New("unexpected status code: %v", resp.Status)
}
var lintResp TemplateLintResp
if err := json.NewDecoder(resp.Body).Decode(&lintResp); err != nil {
return false, err
}
if lintResp.Lint {
return true, nil
}
if lintResp.LintError.Reason != "" {
return false, errorutil.NewWithTag("lint", lintResp.LintError.Reason+" : at line %v", lintResp.LintError.Mark.Line)
}
return false, errorutil.NewWithTag("lint", "at line: %v", lintResp.LintError.Mark.Line)
}
// validateTemplate validates template data using templateman validate api
func validateTemplate(data string) (bool, error) {
resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/validate", tmBaseUrl), "application/x-yaml", strings.NewReader(data))
if err != nil {
return false, err
}
if resp.StatusCode != 200 {
return false, errorutil.New("unexpected status code: %v", resp.Status)
}
var validateResp TemplateResp
if err := json.NewDecoder(resp.Body).Decode(&validateResp); err != nil {
return false, err
}
if validateResp.Validate {
return true, nil
}
if validateResp.ValidateErrorCount > 0 {
if len(validateResp.ValidateError) > 0 {
return false, errorutil.NewWithTag("validate", validateResp.ValidateError[0].Message+": at line %v", validateResp.ValidateError[0].Mark.Line)
}
return false, errorutil.New("validation failed").WithTag("validate")
}
if validateResp.Error.Name != "" {
return false, errorutil.New(validateResp.Error.Name)
}
return false, errorutil.New("template validation failed")
}
// parseAndAddMaxRequests parses and adds max requests to templates
func parseAndAddMaxRequests(catalog catalog.Catalog, path, data string) (string, bool, error) {
template, err := parseTemplate(catalog, path)
if err != nil {
return data, false, err
}
if template.TotalRequests < 1 {
return data, false, nil
}
// Marshal the updated info block back to YAML.
infoBlockStart, infoBlockEnd := getInfoStartEnd(data)
infoBlockOrig := data[infoBlockStart:infoBlockEnd]
infoBlockOrig = strings.TrimRight(infoBlockOrig, "\n")
infoBlock := InfoBlock{}
err = yaml.Unmarshal([]byte(data), &infoBlock)
if err != nil {
return data, false, err
}
// if metadata is nil, create a new map
if infoBlock.Info.Metadata == nil {
infoBlock.Info.Metadata = make(map[string]interface{})
}
// do not update if it is already present and equal
if mr, ok := infoBlock.Info.Metadata["max-request"]; ok && mr.(int) == template.TotalRequests {
return data, false, nil
}
infoBlock.Info.Metadata["max-request"] = template.TotalRequests
var newInfoBlock bytes.Buffer
yamlEncoder := yaml.NewEncoder(&newInfoBlock)
yamlEncoder.SetIndent(yamlIndentSpaces)
err = yamlEncoder.Encode(infoBlock)
if err != nil {
return data, false, err
}
newInfoBlockData := strings.TrimSuffix(newInfoBlock.String(), "\n")
// replace old info block with new info block
newTemplate := strings.ReplaceAll(data, infoBlockOrig, newInfoBlockData)
err = os.WriteFile(path, []byte(newTemplate), 0644)
if err == nil {
return newTemplate, true, nil
}
return newTemplate, false, err
}
// parseTemplate parses a template and returns the template object
func parseTemplate(catalog catalog.Catalog, templatePath string) (*templates.Template, error) {
executorOpts := protocols.ExecutorOptions{
Catalog: catalog,
Options: defaultOpts,
}
reader, err := executorOpts.Catalog.OpenFile(templatePath)
if err != nil {
return nil, err
}
template, err := templates.ParseTemplateFromReader(reader, nil, executorOpts)
if err != nil {
return nil, err
}
return template, nil
}
// find the start and end of the info block
func getInfoStartEnd(data string) (int, int) {
info := strings.Index(data, "info:")
var indices []int
for _, re := range allTagsRegex {
// find the first occurrence of the label
match := re.FindStringIndex(data)
if match != nil {
indices = append(indices, match[0])
}
}
// find the first one after info block
sort.Ints(indices)
return info, indices[0] - 1
}

80
v2/cmd/tmc/types.go Normal file
View File

@ -0,0 +1,80 @@
package main
type Mark struct {
Name string `json:"name,omitempty"`
Position int `json:"position,omitempty"`
Line int `json:"line,omitempty"`
Column int `json:"column,omitempty"`
Snippet string `json:"snippet,omitempty"`
}
type Error struct {
Name string `json:"name"`
Mark Mark `json:"mark"`
}
type LintError struct {
Name string `json:"name,omitempty"`
Reason string `json:"reason,omitempty"`
Mark Mark `json:"mark,omitempty"`
}
type TemplateLintResp struct {
Input string `json:"template_input,omitempty"`
Lint bool `json:"template_lint,omitempty"`
LintError LintError `json:"lint_error,omitempty"`
}
type ValidateError struct {
Location string `json:"location,omitempty"`
Message string `json:"message,omitempty"`
Name string `json:"name,omitempty"`
Argument interface{} `json:"argument,omitempty"`
Stack string `json:"stack,omitempty"`
Mark struct {
Line int `json:"line,omitempty"`
Column int `json:"column,omitempty"`
Pos int `json:"pos,omitempty"`
} `json:"mark,omitempty"`
}
// TemplateResponse from templateman to be used for enhancing and formatting
type TemplateResp struct {
Input string `json:"template_input,omitempty"`
Format bool `json:"template_format,omitempty"`
Updated string `json:"updated_template,omitempty"`
Enhance bool `json:"template_enhance,omitempty"`
Enhanced string `json:"enhanced_template,omitempty"`
Lint bool `json:"template_lint,omitempty"`
LintError LintError `json:"lint_error,omitempty"`
Validate bool `json:"template_validate,omitempty"`
ValidateErrorCount int `json:"validate_error_count,omitempty"`
ValidateError []ValidateError `json:"validate_error,omitempty"`
Error Error `json:"error,omitempty"`
}
// InfoBlock Cloning struct from nuclei as we don't want any validation
type InfoBlock struct {
Info TemplateInfo `yaml:"info"`
}
type TemplateClassification struct {
CvssMetrics string `yaml:"cvss-metrics,omitempty"`
CvssScore float64 `yaml:"cvss-score,omitempty"`
CveId string `yaml:"cve-id,omitempty"`
CweId string `yaml:"cwe-id,omitempty"`
Cpe string `yaml:"cpe,omitempty"`
EpssScore float64 `yaml:"epss-score,omitempty"`
}
type TemplateInfo struct {
Name string `yaml:"name"`
Author string `yaml:"author"`
Severity string `yaml:"severity,omitempty"`
Description string `yaml:"description,omitempty"`
Reference interface{} `yaml:"reference,omitempty"`
Remediation string `yaml:"remediation,omitempty"`
Classification TemplateClassification `yaml:"classification,omitempty"`
Metadata map[string]interface{} `yaml:"metadata,omitempty"`
Tags string `yaml:"tags,omitempty"`
}

View File

@ -72,7 +72,6 @@ require (
github.com/projectdiscovery/gologger v1.1.10
github.com/projectdiscovery/httpx v1.3.0
github.com/projectdiscovery/mapcidr v1.1.2
github.com/projectdiscovery/nvd v1.0.10-0.20230327073015-721181aba1e8
github.com/projectdiscovery/ratelimit v0.0.8
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
github.com/projectdiscovery/sarif v0.0.1

View File

@ -31,7 +31,6 @@ github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd h1:RTWs+wEY
github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd/go.mod h1:AqtPw7WNT0O69k+AbPKWVGYeW94TqgMW/g+Ppc8AZr4=
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 h1:ZK3C5DtzV2nVAQTx5S5jQvMeDqWtD1By5mOoyY/xJek=
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE=
@ -54,7 +53,6 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ=
@ -69,7 +67,6 @@ github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes=
github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
@ -355,7 +352,6 @@ github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXm
github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw=
github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8=
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
github.com/miekg/dns v1.1.54 h1:5jon9mWcb0sFJGpnI99tOMhCPyJ+RPVz5b63MQG0VWI=
github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
@ -431,8 +427,6 @@ github.com/projectdiscovery/mapcidr v1.1.2 h1:Mmq/nPqvVc7fjvH/kJVK0IBOny/LrJIxZ4
github.com/projectdiscovery/mapcidr v1.1.2/go.mod h1:Aoq0x/wJl6KDbtQ8OcPkjIDCqx2iEyx5ty1nzso8wXM=
github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o=
github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk=
github.com/projectdiscovery/nvd v1.0.10-0.20230327073015-721181aba1e8 h1:aDq18tNWbnN5ZM0ADQb+8KB4DEPIGZMXdDmcXyFUoNg=
github.com/projectdiscovery/nvd v1.0.10-0.20230327073015-721181aba1e8/go.mod h1:JiVXOIewstCBMPsO+ZnmI43UXMPJGEE1jwuFVz4ujKM=
github.com/projectdiscovery/ratelimit v0.0.8 h1:K6S/DCr48xNxTXHRmU82wl1mj7j0VrXnAKr8sKTacHI=
github.com/projectdiscovery/ratelimit v0.0.8/go.mod h1:JJAtj8Rd5DNqN5FgwyMHWIi4BHivOw1+8gDrpsBf8Ic=
github.com/projectdiscovery/rawhttp v0.1.13 h1:Xn3NY3SYIk0151K5Qfuvx3tayl2UOoxMuVyYvGT95BA=
@ -450,7 +444,6 @@ github.com/projectdiscovery/tlsx v1.1.0 h1:6L5VKpHaoqvIHN6lH9zi7jIvph1JwYMYZOIpW
github.com/projectdiscovery/tlsx v1.1.0/go.mod h1:C9xTbU2t54Anmvuq+4jxevR5rzqpp6XUUtV7G9J5CTE=
github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak=
github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8=
github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0=
github.com/projectdiscovery/utils v0.0.38 h1:EIAgaP3imfcQY+laxNOU9LXh7VZNAbmiwXsQN0mAxdQ=
github.com/projectdiscovery/utils v0.0.38/go.mod h1:5+WAxSV7yGl6SDCtR1qiOyiEMCIo3jIff+A5OiYTCgM=
github.com/projectdiscovery/wappalyzergo v0.0.94 h1:IVRskuU95MajWCKYgvH5L67+MXDOWJDWSeBD61OsS/A=
@ -468,7 +461,6 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/sashabaranov/go-openai v1.9.1 h1:3N52HkJKo9Zlo/oe1AVv5ZkCOny0ra58/ACvAxkN3MM=
@ -599,7 +591,6 @@ github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhu
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30=
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is=
github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw=
github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 h1:QuLjRpIBjqene8VvB+VhQ4eTcQGCQ7JDuk0/Fp4sLLw=
github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw=
go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
@ -610,7 +601,6 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
@ -636,7 +626,6 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
golang.org/x/exp v0.0.0-20221019170559-20944726eadf/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@ -646,7 +635,6 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@ -660,13 +648,11 @@ golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
@ -717,7 +703,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@ -730,7 +715,6 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
@ -740,7 +724,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
@ -755,7 +738,6 @@ golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDq
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM=