Feat template update improvements (#3675)

* path modification of official templates

* fix deprecated paths counter

* add reset flag to nuclei

* bug fix: deprecated path counter

* ignore meta files

* purge empty dirs

* fix lint error

README.md

@@ -143,8 +143,8 @@ FILTERING:
    -em, -exclude-matchers string[]    template matchers to exclude in result
    -s, -severity value[]              templates to run based on severity. Possible values: info, low, medium, high, critical, unknown
    -es, -exclude-severity value[]     templates to exclude based on severity. Possible values: info, low, medium, high, critical, unknown
-   -pt, -type value[]                 templates to run based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois
-   -ept, -exclude-type value[]        templates to exclude based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois
+   -pt, -type value[]                 templates to run based on protocol type. Possible values: dns, file, http, headless, tcp, workflow, ssl, websocket, whois
+   -ept, -exclude-type value[]        templates to exclude based on protocol type. Possible values: dns, file, http, headless, tcp, workflow, ssl, websocket, whois
    -tc, -template-condition string[]  templates to run based on expression condition
 
 OUTPUT:
@@ -153,7 +153,7 @@ OUTPUT:
    -srd, -store-resp-dir string  store all request/response passed through nuclei to custom directory (default "output")
    -silent                       display findings only
    -nc, -no-color                disable output content coloring (ANSI escape codes)
-   -j -jsonl                     write output in JSONL(ines) format
+   -j, -jsonl                    write output in JSONL(ines) format
    -irr, -include-rr             include request/response pairs in the JSONL output (for findings only)
    -nm, -no-meta                 disable printing result metadata in cli output
    -ts, -timestamp               enables printing timestamp in cli output
@@ -161,8 +161,8 @@ OUTPUT:
    -ms, -matcher-status          display match failure status
    -me, -markdown-export string  directory to export results in markdown format
    -se, -sarif-export string     file to export results in SARIF format
-   -je, -json-export string      file to export results in JSON format as a JSON array. This can be memory intensive in larger scans
-   -jle, -jsonl-export string    file to export results in JSONL(ine) format as a list of line-delimited JSON objects
+   -je, -json-export string      file to export results in JSON format
+   -jle, -jsonl-export string    file to export results in JSONL(ine) format
 
 CONFIGURATIONS:
    -config string                 path to the nuclei configuration file
@@ -192,6 +192,7 @@ CONFIGURATIONS:
    -config-directory string       override the default config path ($home/.config)
    -rsr, -response-size-read int  max response size to read in bytes (default 10485760)
    -rss, -response-size-save int  max response size to read in bytes (default 1048576)
+   -reset                         reset removes all nuclei configuration and data files (including nuclei-templates)
 
 INTERACTSH:
    -iserver, -interactsh-server string  interactsh server url for self-hosted instance (default: oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me)
@@ -233,7 +234,7 @@ OPTIMIZATIONS:
    -project-path string                set a specific project path (default "/tmp")
    -spm, -stop-at-first-match          stop processing HTTP requests after the first match (may break template/workflow logic)
    -stream                             stream mode - start elaborating without sorting the input
-   -ss, -scan-strategy value           strategy to use while scanning(auto/host-spray/template-spray) (default 0)
+   -ss, -scan-strategy value           strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout duration  timeout on input read (default 3m0s)
    -nh, -no-httpx                      disable httpx probing for non-url input
    -no-stdin                           disable stdin processing

v2/cmd/nuclei/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bufio"
 	"fmt"
 	"os"
 	"os/signal"
@@ -207,6 +208,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVar(&options.CustomConfigDir, "config-directory", "", "override the default config path ($home/.config)"),
 		flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 10*1024*1024, "max response size to read in bytes"),
 		flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", 1*1024*1024, "max response size to read in bytes"),
+		flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
 	)
 
 	flagSet.CreateGroup("interactsh", "interactsh",
@@ -427,6 +429,46 @@ func printTemplateVersion() {
 	os.Exit(0)
 }
 
+func resetCallback() {
+	warning := fmt.Sprintf(`
+Using '-reset' will delete all nuclei configurations files and all nuclei-templates
+
+Following files will be deleted:
+1. All Config + Resumes files at %v
+2. All nuclei-templates at %v
+
+Note: Make sure you have backup of your custom nuclei-templates before proceeding
+
+`, config.DefaultConfig.GetConfigDir(), config.DefaultConfig.TemplatesDirectory)
+	gologger.Print().Msg(warning)
+	reader := bufio.NewReader(os.Stdin)
+	for {
+		fmt.Print("Are you sure you want to continue? [y/n]: ")
+		resp, err := reader.ReadString('\n')
+		if err != nil {
+			gologger.Fatal().Msgf("could not read response: %s", err)
+		}
+		resp = strings.TrimSpace(resp)
+		if strings.EqualFold(resp, "y") || strings.EqualFold(resp, "yes") {
+			break
+		}
+		if strings.EqualFold(resp, "n") || strings.EqualFold(resp, "no") || resp == "" {
+			fmt.Println("Exiting...")
+			os.Exit(0)
+		}
+	}
+	err := os.RemoveAll(config.DefaultConfig.GetConfigDir())
+	if err != nil {
+		gologger.Fatal().Msgf("could not delete config dir: %s", err)
+	}
+	err = os.RemoveAll(config.DefaultConfig.TemplatesDirectory)
+	if err != nil {
+		gologger.Fatal().Msgf("could not delete templates dir: %s", err)
+	}
+	gologger.Info().Msgf("Successfully deleted all nuclei configurations files and nuclei-templates")
+	os.Exit(0)
+}
+
 func init() {
 	// print stacktrace of errors in debug mode
 	if strings.EqualFold(os.Getenv("DEBUG"), "true") {

v2/internal/installer/template.go

@@ -229,23 +229,53 @@ func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.File
 
 // writeChecksumFileInDir is actual method responsible for writing all templates to directory
 func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownloader, dir string) error {
+	LocaltemplatesIndex, err := config.GetNucleiTemplatesIndex()
+	if err != nil {
+		gologger.Warning().Msgf("failed to get local nuclei-templates index: %s", err)
+		if LocaltemplatesIndex == nil {
+			LocaltemplatesIndex = map[string]string{} // no-op
+		}
+	}
+
 	callbackFunc := func(uri string, f fs.FileInfo, r io.Reader) error {
 		writePath := t.getAbsoluteFilePath(dir, uri, f)
 		if writePath == "" {
 			// skip writing file
 			return nil
 		}
+
 		bin, err := io.ReadAll(r)
 		if err != nil {
 			// if error occurs, iteration also stops
 			return errorutil.NewWithErr(err).Msgf("failed to read file %s", uri)
 		}
+		// TODO: It might be better to just download index file from nuclei templates repo
+		// instead of creating it from scratch
+		id, _ := config.GetTemplateIDFromReader(bytes.NewReader(bin), uri)
+		if id != "" {
+			// based on template id, check if we are updating path of official nuclei template
+			if oldPath, ok := LocaltemplatesIndex[id]; ok {
+				if oldPath != writePath {
+					// write new template at new path and delete old template
+					if err := os.WriteFile(writePath, bin, f.Mode()); err != nil {
+						return errorutil.NewWithErr(err).Msgf("failed to write file %s", uri)
+					}
+					// after successful write, remove old template
+					if err := os.Remove(oldPath); err != nil {
+						gologger.Warning().Msgf("failed to remove old template %s: %s", oldPath, err)
+					}
+					return nil
+				}
+			}
+		}
+		// no change in template Path of official templates
 		return os.WriteFile(writePath, bin, f.Mode())
 	}
-	err := ghrd.DownloadSourceWithCallback(!HideProgressBar, callbackFunc)
+	err = ghrd.DownloadSourceWithCallback(!HideProgressBar, callbackFunc)
 	if err != nil {
 		return errorutil.NewWithErr(err).Msgf("failed to download templates")
 	}
+
 	if err := config.DefaultConfig.WriteTemplatesConfig(); err != nil {
 		return errorutil.NewWithErr(err).Msgf("failed to write templates config")
 	}
@@ -259,6 +289,20 @@ func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownlo
 		return errorutil.NewWithErr(err).Msgf("failed to update templates version")
 	}
 
+	PurgeEmptyDirectories(dir)
+
+	// generate index of all templates
+	_ = os.Remove(config.DefaultConfig.GetTemplateIndexFilePath())
+
+	index, err := config.GetNucleiTemplatesIndex()
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("failed to get nuclei templates index")
+	}
+
+	if err = config.DefaultConfig.WriteTemplatesIndex(index); err != nil {
+		return errorutil.NewWithErr(err).Msgf("failed to write nuclei templates index")
+	}
+
 	// after installation create and write checksums to .checksum file
 	return t.writeChecksumFileInDir(dir)
 }

v2/internal/installer/util.go

@@ -5,7 +5,11 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/fs"
 	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
 
 	"github.com/Masterminds/semver/v3"
 	"github.com/projectdiscovery/gologger"
@@ -67,3 +71,35 @@ func getNewAdditionsFileFromGithub(version string) ([]string, error) {
 	}
 	return templatesList, nil
 }
+
+func PurgeEmptyDirectories(dir string) {
+	alldirs := []string{}
+	_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+		if d.IsDir() {
+			alldirs = append(alldirs, path)
+		}
+		return nil
+	})
+	// sort in ascending order
+	sort.Strings(alldirs)
+	// reverse the order
+	sort.Sort(sort.Reverse(sort.StringSlice(alldirs)))
+
+	for _, d := range alldirs {
+		if isEmptyDir(d) {
+			_ = os.RemoveAll(d)
+		}
+	}
+}
+
+func isEmptyDir(dir string) bool {
+	hasFiles := false
+	_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+		if !d.IsDir() {
+			hasFiles = true
+			return io.EOF
+		}
+		return nil
+	})
+	return !hasFiles
+}

v2/pkg/catalog/config/constants.go

@@ -11,6 +11,7 @@ const (
 	NucleiTemplatesDirName           = "nuclei-templates"
 	OfficialNucleiTeamplatesRepoName = "nuclei-templates"
 	NucleiIgnoreFileName             = ".nuclei-ignore"
+	NucleiTemplatesIndexFileName     = ".templates-index" // contains index of official nuclei templates
 	NucleiTemplatesCheckSumFileName  = ".checksum"
 	NewTemplateAdditionsFileName     = ".new-additions"
 	CLIConifgFileName                = "config.yaml"

v2/pkg/catalog/config/nucleiconfig.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"bytes"
 	"crypto/md5"
 	"encoding/json"
 	"fmt"
@@ -123,6 +124,10 @@ func (c *Config) GetIgnoreFilePath() string {
 	return filepath.Join(c.configDir, NucleiIgnoreFileName)
 }
 
+func (c *Config) GetTemplateIndexFilePath() string {
+	return filepath.Join(c.TemplatesDirectory, NucleiTemplatesIndexFileName)
+}
+
 // GetTemplatesConfigFilePath returns checksum file path of nuclei templates
 func (c *Config) GetChecksumFilePath() string {
 	return filepath.Join(c.TemplatesDirectory, NucleiTemplatesCheckSumFileName)
@@ -237,6 +242,16 @@ func (c *Config) WriteTemplatesConfig() error {
 	return nil
 }
 
+// WriteTemplatesIndex writes the nuclei templates index file
+func (c *Config) WriteTemplatesIndex(index map[string]string) error {
+	indexFile := c.GetTemplateIndexFilePath()
+	var buff bytes.Buffer
+	for k, v := range index {
+		_, _ = buff.WriteString(k + "," + v + "\n")
+	}
+	return os.WriteFile(indexFile, buff.Bytes(), 0600)
+}
+
 // getTemplatesConfigFilePath returns configDir/.templates-config.json file path
 func (c *Config) getTemplatesConfigFilePath() string {
 	return filepath.Join(c.configDir, TemplateConfigFileName)

v2/pkg/catalog/config/template.go

@@ -1,13 +1,20 @@
 package config
 
 import (
+	"encoding/csv"
+	"io"
+	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions"
+	fileutil "github.com/projectdiscovery/utils/file"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
+var knownConfigFiles = []string{"cves.json", "contributors.json", "TEMPLATES-STATS.json"}
+
 // TemplateFormat
 type TemplateFormat uint8
 
@@ -38,5 +45,82 @@ func GetSupportTemplateFileExtensions() []string {
 // isTemplate is a callback function used by goflags to decide if given file should be read
 // if it is not a nuclei-template file only then file is read
 func IsTemplate(filename string) bool {
+	if stringsutil.ContainsAny(filename, knownConfigFiles...) {
+		return false
+	}
 	return stringsutil.EqualFoldAny(filepath.Ext(filename), GetSupportTemplateFileExtensions()...)
 }
+
+type template struct {
+	ID string `json:"id" yaml:"id"`
+}
+
+// GetTemplateIDFromReader returns template id from reader
+func GetTemplateIDFromReader(data io.Reader, filename string) (string, error) {
+	var t template
+	var err error
+	switch GetTemplateFormatFromExt(filename) {
+	case YAML:
+		err = fileutil.UnmarshalFromReader(fileutil.YAML, data, &t)
+	case JSON:
+		err = fileutil.UnmarshalFromReader(fileutil.JSON, data, &t)
+	}
+	return t.ID, err
+}
+
+func getTemplateID(filePath string) (string, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return "", err
+	}
+
+	defer file.Close()
+	return GetTemplateIDFromReader(file, filePath)
+}
+
+// GetTemplatesIndexFile returns map[template-id]: template-file-path
+func GetNucleiTemplatesIndex() (map[string]string, error) {
+	indexFile := DefaultConfig.GetTemplateIndexFilePath()
+	index := map[string]string{}
+	if fileutil.FileExists(indexFile) {
+		f, err := os.Open(indexFile)
+		if err == nil {
+			csvReader := csv.NewReader(f)
+			records, err := csvReader.ReadAll()
+			if err == nil {
+				for _, v := range records {
+					if len(v) >= 2 {
+						index[v[0]] = v[1]
+					}
+				}
+				return index, nil
+			}
+		}
+		gologger.Error().Msgf("failed to read index file creating new one: %v", err)
+	}
+
+	ignoreDirs := DefaultConfig.GetAllCustomTemplateDirs()
+
+	// empty index if templates are not installed
+	if !fileutil.FolderExists(DefaultConfig.TemplatesDirectory) {
+		return index, nil
+	}
+	err := filepath.WalkDir(DefaultConfig.TemplatesDirectory, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			gologger.Verbose().Msgf("failed to walk path=%v err=%v", path, err)
+			return nil
+		}
+		if d.IsDir() || !IsTemplate(path) || stringsutil.ContainsAny(path, ignoreDirs...) {
+			return nil
+		}
+		// get template id from file
+		id, err := getTemplateID(path)
+		if err != nil || id == "" {
+			gologger.Verbose().Msgf("failed to get template id from file=%v got id=%v err=%v", path, id, err)
+			return nil
+		}
+		index[id] = path
+		return nil
+	})
+	return index, err
+}

v2/pkg/catalog/disk/find.go

@@ -80,7 +80,7 @@ func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) {
 
 	// try to handle deprecated template paths
 	absPath := BackwardsCompatiblePaths(c.templatesDirectory, target)
-	if absPath != target {
+	if absPath != target && strings.TrimPrefix(absPath, c.templatesDirectory+string(filepath.Separator)) != target {
 		deprecatedPathsCounter++
 	}
 
@@ -212,10 +212,9 @@ func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]
 // Unless mode is silent warning message is printed
 func PrintDeprecatedPathsMsgIfApplicable(isSilent bool) {
 	if !updateutils.IsOutdated("v9.4.3", config.DefaultConfig.TemplateVersion) {
-		// template version is not older than 9.4.3
 		return
 	}
 	if deprecatedPathsCounter > 0 && !isSilent {
-		gologger.Print().Msgf("[%v] Found %v templates loaded with deprecated paths, update before v2.9.5 for continued support.\n", aurora.Yellow("WRN").String(), deprecatedPathsCounter)
+		gologger.Print().Msgf("[%v] Found %v template[s] loaded with deprecated paths, update before v2.9.5 for continued support.\n", aurora.Yellow("WRN").String(), deprecatedPathsCounter)
 	}
 }