diff --git a/README.md b/README.md index d7a70809..6642942e 100644 --- a/README.md +++ b/README.md @@ -143,8 +143,8 @@ FILTERING: -em, -exclude-matchers string[] template matchers to exclude in result -s, -severity value[] templates to run based on severity. Possible values: info, low, medium, high, critical, unknown -es, -exclude-severity value[] templates to exclude based on severity. Possible values: info, low, medium, high, critical, unknown - -pt, -type value[] templates to run based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois - -ept, -exclude-type value[] templates to exclude based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois + -pt, -type value[] templates to run based on protocol type. Possible values: dns, file, http, headless, tcp, workflow, ssl, websocket, whois + -ept, -exclude-type value[] templates to exclude based on protocol type. Possible values: dns, file, http, headless, tcp, workflow, ssl, websocket, whois -tc, -template-condition string[] templates to run based on expression condition OUTPUT: @@ -153,7 +153,7 @@ OUTPUT: -srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output") -silent display findings only -nc, -no-color disable output content coloring (ANSI escape codes) - -j -jsonl write output in JSONL(ines) format + -j, -jsonl write output in JSONL(ines) format -irr, -include-rr include request/response pairs in the JSONL output (for findings only) -nm, -no-meta disable printing result metadata in cli output -ts, -timestamp enables printing timestamp in cli output @@ -161,8 +161,8 @@ OUTPUT: -ms, -matcher-status display match failure status -me, -markdown-export string directory to export results in markdown format -se, -sarif-export string file to export results in SARIF format - -je, -json-export string file to export results in JSON format as a JSON array. This can be memory intensive in larger scans - -jle, -jsonl-export string file to export results in JSONL(ine) format as a list of line-delimited JSON objects + -je, -json-export string file to export results in JSON format + -jle, -jsonl-export string file to export results in JSONL(ine) format CONFIGURATIONS: -config string path to the nuclei configuration file @@ -192,6 +192,7 @@ CONFIGURATIONS: -config-directory string override the default config path ($home/.config) -rsr, -response-size-read int max response size to read in bytes (default 10485760) -rss, -response-size-save int max response size to read in bytes (default 1048576) + -reset reset removes all nuclei configuration and data files (including nuclei-templates) INTERACTSH: -iserver, -interactsh-server string interactsh server url for self-hosted instance (default: oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me) @@ -233,7 +234,7 @@ OPTIMIZATIONS: -project-path string set a specific project path (default "/tmp") -spm, -stop-at-first-match stop processing HTTP requests after the first match (may break template/workflow logic) -stream stream mode - start elaborating without sorting the input - -ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default 0) + -ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto) -irt, -input-read-timeout duration timeout on input read (default 3m0s) -nh, -no-httpx disable httpx probing for non-url input -no-stdin disable stdin processing diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index aec4edef..01e5df93 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "fmt" "os" "os/signal" @@ -207,6 +208,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVar(&options.CustomConfigDir, "config-directory", "", "override the default config path ($home/.config)"), flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 10*1024*1024, "max response size to read in bytes"), flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", 1*1024*1024, "max response size to read in bytes"), + flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"), ) flagSet.CreateGroup("interactsh", "interactsh", @@ -427,6 +429,46 @@ func printTemplateVersion() { os.Exit(0) } +func resetCallback() { + warning := fmt.Sprintf(` +Using '-reset' will delete all nuclei configurations files and all nuclei-templates + +Following files will be deleted: +1. All Config + Resumes files at %v +2. All nuclei-templates at %v + +Note: Make sure you have backup of your custom nuclei-templates before proceeding + +`, config.DefaultConfig.GetConfigDir(), config.DefaultConfig.TemplatesDirectory) + gologger.Print().Msg(warning) + reader := bufio.NewReader(os.Stdin) + for { + fmt.Print("Are you sure you want to continue? [y/n]: ") + resp, err := reader.ReadString('\n') + if err != nil { + gologger.Fatal().Msgf("could not read response: %s", err) + } + resp = strings.TrimSpace(resp) + if strings.EqualFold(resp, "y") || strings.EqualFold(resp, "yes") { + break + } + if strings.EqualFold(resp, "n") || strings.EqualFold(resp, "no") || resp == "" { + fmt.Println("Exiting...") + os.Exit(0) + } + } + err := os.RemoveAll(config.DefaultConfig.GetConfigDir()) + if err != nil { + gologger.Fatal().Msgf("could not delete config dir: %s", err) + } + err = os.RemoveAll(config.DefaultConfig.TemplatesDirectory) + if err != nil { + gologger.Fatal().Msgf("could not delete templates dir: %s", err) + } + gologger.Info().Msgf("Successfully deleted all nuclei configurations files and nuclei-templates") + os.Exit(0) +} + func init() { // print stacktrace of errors in debug mode if strings.EqualFold(os.Getenv("DEBUG"), "true") { diff --git a/v2/internal/installer/template.go b/v2/internal/installer/template.go index 9a0c8492..ea4f5757 100644 --- a/v2/internal/installer/template.go +++ b/v2/internal/installer/template.go @@ -229,23 +229,53 @@ func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.File // writeChecksumFileInDir is actual method responsible for writing all templates to directory func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownloader, dir string) error { + LocaltemplatesIndex, err := config.GetNucleiTemplatesIndex() + if err != nil { + gologger.Warning().Msgf("failed to get local nuclei-templates index: %s", err) + if LocaltemplatesIndex == nil { + LocaltemplatesIndex = map[string]string{} // no-op + } + } + callbackFunc := func(uri string, f fs.FileInfo, r io.Reader) error { writePath := t.getAbsoluteFilePath(dir, uri, f) if writePath == "" { // skip writing file return nil } + bin, err := io.ReadAll(r) if err != nil { // if error occurs, iteration also stops return errorutil.NewWithErr(err).Msgf("failed to read file %s", uri) } + // TODO: It might be better to just download index file from nuclei templates repo + // instead of creating it from scratch + id, _ := config.GetTemplateIDFromReader(bytes.NewReader(bin), uri) + if id != "" { + // based on template id, check if we are updating path of official nuclei template + if oldPath, ok := LocaltemplatesIndex[id]; ok { + if oldPath != writePath { + // write new template at new path and delete old template + if err := os.WriteFile(writePath, bin, f.Mode()); err != nil { + return errorutil.NewWithErr(err).Msgf("failed to write file %s", uri) + } + // after successful write, remove old template + if err := os.Remove(oldPath); err != nil { + gologger.Warning().Msgf("failed to remove old template %s: %s", oldPath, err) + } + return nil + } + } + } + // no change in template Path of official templates return os.WriteFile(writePath, bin, f.Mode()) } - err := ghrd.DownloadSourceWithCallback(!HideProgressBar, callbackFunc) + err = ghrd.DownloadSourceWithCallback(!HideProgressBar, callbackFunc) if err != nil { return errorutil.NewWithErr(err).Msgf("failed to download templates") } + if err := config.DefaultConfig.WriteTemplatesConfig(); err != nil { return errorutil.NewWithErr(err).Msgf("failed to write templates config") } @@ -259,6 +289,20 @@ func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownlo return errorutil.NewWithErr(err).Msgf("failed to update templates version") } + PurgeEmptyDirectories(dir) + + // generate index of all templates + _ = os.Remove(config.DefaultConfig.GetTemplateIndexFilePath()) + + index, err := config.GetNucleiTemplatesIndex() + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to get nuclei templates index") + } + + if err = config.DefaultConfig.WriteTemplatesIndex(index); err != nil { + return errorutil.NewWithErr(err).Msgf("failed to write nuclei templates index") + } + // after installation create and write checksums to .checksum file return t.writeChecksumFileInDir(dir) } diff --git a/v2/internal/installer/util.go b/v2/internal/installer/util.go index 72856266..c9b1b7c4 100644 --- a/v2/internal/installer/util.go +++ b/v2/internal/installer/util.go @@ -5,7 +5,11 @@ import ( "bytes" "fmt" "io" + "io/fs" "net/http" + "os" + "path/filepath" + "sort" "github.com/Masterminds/semver/v3" "github.com/projectdiscovery/gologger" @@ -67,3 +71,35 @@ func getNewAdditionsFileFromGithub(version string) ([]string, error) { } return templatesList, nil } + +func PurgeEmptyDirectories(dir string) { + alldirs := []string{} + _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if d.IsDir() { + alldirs = append(alldirs, path) + } + return nil + }) + // sort in ascending order + sort.Strings(alldirs) + // reverse the order + sort.Sort(sort.Reverse(sort.StringSlice(alldirs))) + + for _, d := range alldirs { + if isEmptyDir(d) { + _ = os.RemoveAll(d) + } + } +} + +func isEmptyDir(dir string) bool { + hasFiles := false + _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if !d.IsDir() { + hasFiles = true + return io.EOF + } + return nil + }) + return !hasFiles +} diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go index 4fc82a4b..1ef4b1d1 100644 --- a/v2/pkg/catalog/config/constants.go +++ b/v2/pkg/catalog/config/constants.go @@ -11,6 +11,7 @@ const ( NucleiTemplatesDirName = "nuclei-templates" OfficialNucleiTeamplatesRepoName = "nuclei-templates" NucleiIgnoreFileName = ".nuclei-ignore" + NucleiTemplatesIndexFileName = ".templates-index" // contains index of official nuclei templates NucleiTemplatesCheckSumFileName = ".checksum" NewTemplateAdditionsFileName = ".new-additions" CLIConifgFileName = "config.yaml" diff --git a/v2/pkg/catalog/config/nucleiconfig.go b/v2/pkg/catalog/config/nucleiconfig.go index 59e0646c..de2e5044 100644 --- a/v2/pkg/catalog/config/nucleiconfig.go +++ b/v2/pkg/catalog/config/nucleiconfig.go @@ -1,6 +1,7 @@ package config import ( + "bytes" "crypto/md5" "encoding/json" "fmt" @@ -123,6 +124,10 @@ func (c *Config) GetIgnoreFilePath() string { return filepath.Join(c.configDir, NucleiIgnoreFileName) } +func (c *Config) GetTemplateIndexFilePath() string { + return filepath.Join(c.TemplatesDirectory, NucleiTemplatesIndexFileName) +} + // GetTemplatesConfigFilePath returns checksum file path of nuclei templates func (c *Config) GetChecksumFilePath() string { return filepath.Join(c.TemplatesDirectory, NucleiTemplatesCheckSumFileName) @@ -237,6 +242,16 @@ func (c *Config) WriteTemplatesConfig() error { return nil } +// WriteTemplatesIndex writes the nuclei templates index file +func (c *Config) WriteTemplatesIndex(index map[string]string) error { + indexFile := c.GetTemplateIndexFilePath() + var buff bytes.Buffer + for k, v := range index { + _, _ = buff.WriteString(k + "," + v + "\n") + } + return os.WriteFile(indexFile, buff.Bytes(), 0600) +} + // getTemplatesConfigFilePath returns configDir/.templates-config.json file path func (c *Config) getTemplatesConfigFilePath() string { return filepath.Join(c.configDir, TemplateConfigFileName) diff --git a/v2/pkg/catalog/config/template.go b/v2/pkg/catalog/config/template.go index 806a67f0..31741c0c 100644 --- a/v2/pkg/catalog/config/template.go +++ b/v2/pkg/catalog/config/template.go @@ -1,13 +1,20 @@ package config import ( + "encoding/csv" + "io" + "os" "path/filepath" "strings" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions" + fileutil "github.com/projectdiscovery/utils/file" stringsutil "github.com/projectdiscovery/utils/strings" ) +var knownConfigFiles = []string{"cves.json", "contributors.json", "TEMPLATES-STATS.json"} + // TemplateFormat type TemplateFormat uint8 @@ -38,5 +45,82 @@ func GetSupportTemplateFileExtensions() []string { // isTemplate is a callback function used by goflags to decide if given file should be read // if it is not a nuclei-template file only then file is read func IsTemplate(filename string) bool { + if stringsutil.ContainsAny(filename, knownConfigFiles...) { + return false + } return stringsutil.EqualFoldAny(filepath.Ext(filename), GetSupportTemplateFileExtensions()...) } + +type template struct { + ID string `json:"id" yaml:"id"` +} + +// GetTemplateIDFromReader returns template id from reader +func GetTemplateIDFromReader(data io.Reader, filename string) (string, error) { + var t template + var err error + switch GetTemplateFormatFromExt(filename) { + case YAML: + err = fileutil.UnmarshalFromReader(fileutil.YAML, data, &t) + case JSON: + err = fileutil.UnmarshalFromReader(fileutil.JSON, data, &t) + } + return t.ID, err +} + +func getTemplateID(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", err + } + + defer file.Close() + return GetTemplateIDFromReader(file, filePath) +} + +// GetTemplatesIndexFile returns map[template-id]: template-file-path +func GetNucleiTemplatesIndex() (map[string]string, error) { + indexFile := DefaultConfig.GetTemplateIndexFilePath() + index := map[string]string{} + if fileutil.FileExists(indexFile) { + f, err := os.Open(indexFile) + if err == nil { + csvReader := csv.NewReader(f) + records, err := csvReader.ReadAll() + if err == nil { + for _, v := range records { + if len(v) >= 2 { + index[v[0]] = v[1] + } + } + return index, nil + } + } + gologger.Error().Msgf("failed to read index file creating new one: %v", err) + } + + ignoreDirs := DefaultConfig.GetAllCustomTemplateDirs() + + // empty index if templates are not installed + if !fileutil.FolderExists(DefaultConfig.TemplatesDirectory) { + return index, nil + } + err := filepath.WalkDir(DefaultConfig.TemplatesDirectory, func(path string, d os.DirEntry, err error) error { + if err != nil { + gologger.Verbose().Msgf("failed to walk path=%v err=%v", path, err) + return nil + } + if d.IsDir() || !IsTemplate(path) || stringsutil.ContainsAny(path, ignoreDirs...) { + return nil + } + // get template id from file + id, err := getTemplateID(path) + if err != nil || id == "" { + gologger.Verbose().Msgf("failed to get template id from file=%v got id=%v err=%v", path, id, err) + return nil + } + index[id] = path + return nil + }) + return index, err +} diff --git a/v2/pkg/catalog/disk/find.go b/v2/pkg/catalog/disk/find.go index 4a33c6d3..7b0f4af0 100644 --- a/v2/pkg/catalog/disk/find.go +++ b/v2/pkg/catalog/disk/find.go @@ -80,7 +80,7 @@ func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) { // try to handle deprecated template paths absPath := BackwardsCompatiblePaths(c.templatesDirectory, target) - if absPath != target { + if absPath != target && strings.TrimPrefix(absPath, c.templatesDirectory+string(filepath.Separator)) != target { deprecatedPathsCounter++ } @@ -212,10 +212,9 @@ func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string] // Unless mode is silent warning message is printed func PrintDeprecatedPathsMsgIfApplicable(isSilent bool) { if !updateutils.IsOutdated("v9.4.3", config.DefaultConfig.TemplateVersion) { - // template version is not older than 9.4.3 return } if deprecatedPathsCounter > 0 && !isSilent { - gologger.Print().Msgf("[%v] Found %v templates loaded with deprecated paths, update before v2.9.5 for continued support.\n", aurora.Yellow("WRN").String(), deprecatedPathsCounter) + gologger.Print().Msgf("[%v] Found %v template[s] loaded with deprecated paths, update before v2.9.5 for continued support.\n", aurora.Yellow("WRN").String(), deprecatedPathsCounter) } }