mirror of https://github.com/daffainfo/nuclei.git
fix relative path issue + remove residual code (#4284)
* fix relative path issue + remove residual code * use template dir in templateFS * fix dir relative path issue * print metrics server address in verbose mode * add timeout for downloading binary & templates * update stats & metrics docs * add template-id loader integration testdev
parent
ad3d33964c
commit
5c9af62037
|
@ -19,9 +19,9 @@ pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser
|
|||
.gitignore
|
||||
pkg/js/devtools/bindgen/cmd/bindgen
|
||||
pkg/js/devtools/jsdocgen/jsdocgen
|
||||
./bindgen
|
||||
./jsdocgen
|
||||
./scrapefuncs
|
||||
*.DS_Store
|
||||
pkg/protocols/headless/engine/.cache
|
||||
./nuclei
|
||||
/nuclei
|
||||
/bindgen
|
||||
/jsdocgen
|
||||
/scrapefuncs
|
||||
|
|
|
@ -10,6 +10,7 @@ import (
|
|||
"github.com/julienschmidt/httprouter"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
|
||||
errorutil "github.com/projectdiscovery/utils/errors"
|
||||
permissionutil "github.com/projectdiscovery/utils/permission"
|
||||
)
|
||||
|
||||
|
@ -20,6 +21,7 @@ var loaderTestcases = []TestCaseInfo{
|
|||
{Path: "loader/nonexistent-template-list.yaml", TestCase: &nonExistentTemplateList{}},
|
||||
{Path: "loader/nonexistent-workflow-list.yaml", TestCase: &nonExistentWorkflowList{}},
|
||||
{Path: "loader/template-list-not-allowed.yaml", TestCase: &remoteTemplateListNotAllowed{}},
|
||||
{Path: "loader/load-template-with-id", TestCase: &loadTemplateWithID{}},
|
||||
}
|
||||
|
||||
type remoteTemplateList struct{}
|
||||
|
@ -193,3 +195,13 @@ func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error
|
|||
|
||||
return nil
|
||||
}
|
||||
|
||||
type loadTemplateWithID struct{}
|
||||
|
||||
func (h *loadTemplateWithID) Execute(nooop string) error {
|
||||
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", "scanme.sh", "-id", "self-signed-ssl")
|
||||
if err != nil {
|
||||
return errorutil.NewWithErr(err).Msgf("failed to load template with id")
|
||||
}
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
|
|
@ -32,6 +32,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/monitor"
|
||||
errorutil "github.com/projectdiscovery/utils/errors"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
updateutils "github.com/projectdiscovery/utils/update"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -158,6 +159,10 @@ func main() {
|
|||
}
|
||||
|
||||
func readConfig() *goflags.FlagSet {
|
||||
|
||||
// when true updates nuclei binary to latest version
|
||||
var updateNucleiBinary bool
|
||||
|
||||
flagSet := goflags.NewFlagSet()
|
||||
flagSet.CaseSensitive = true
|
||||
flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing
|
||||
|
@ -342,7 +347,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
|||
)
|
||||
|
||||
flagSet.CreateGroup("update", "Update",
|
||||
flagSet.CallbackVarP(runner.NucleiToolUpdateCallback, "update", "up", "update nuclei engine to the latest released version"),
|
||||
flagSet.BoolVarP(&updateNucleiBinary, "update", "up", false, "update nuclei engine to the latest released version"),
|
||||
flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update nuclei-templates to latest released version"),
|
||||
flagSet.StringVarP(&options.NewTemplatesDirectory, "update-template-dir", "ud", "", "custom directory to install / update nuclei-templates"),
|
||||
flagSet.CallbackVarP(disableUpdatesCallback, "disable-update-check", "duc", "disable automatic nuclei/templates update check"),
|
||||
|
@ -412,6 +417,14 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
|
|||
installer.HideReleaseNotes = false
|
||||
}
|
||||
|
||||
if options.Timeout > 30 {
|
||||
// default github binary/template download timeout is 30 sec
|
||||
updateutils.DownloadUpdateTimeout = time.Duration(options.Timeout) * time.Second
|
||||
}
|
||||
if updateNucleiBinary {
|
||||
runner.NucleiToolUpdateCallback()
|
||||
}
|
||||
|
||||
if options.LeaveDefaultPorts {
|
||||
http.LeaveDefaultPorts = true
|
||||
}
|
||||
|
|
|
@ -402,10 +402,15 @@ STATISTICS:
|
|||
-stats display statistics about the running scan
|
||||
-sj, -stats-json write statistics data to an output file in JSONL(ines) format
|
||||
-si, -stats-interval int number of seconds to wait between showing a statistics update (default 5)
|
||||
-m, -metrics expose nuclei metrics on a port
|
||||
-mp, -metrics-port int port to expose nuclei metrics on (default 9092)
|
||||
```
|
||||
|
||||
<Tip>
|
||||
From Nuclei v3.0.0 `-metrics` port has been removed and merged with `-stats`
|
||||
when using `-stats` flag metrics will be by default available at `localhost:9092/metrics`
|
||||
and metrics-port can be configured by `-metrics-port` flag
|
||||
</Tip>
|
||||
|
||||
### Rate **Limits**
|
||||
|
||||
Nuclei have multiple rate limit controls for multiple factors, including a number of templates to execute in parallel, a number of hosts to be scanned in parallel for each template, and the global number of request / per second you wanted to make/limit using nuclei, here is an example of each flag with description.
|
||||
|
|
|
@ -17,7 +17,7 @@ const (
|
|||
CLIConfigFileName = "config.yaml"
|
||||
ReportingConfigFilename = "reporting-config.yaml"
|
||||
// Version is the current version of nuclei
|
||||
Version = `v3.0.1`
|
||||
Version = `v3.0.2-dev`
|
||||
// Directory Names of custom templates
|
||||
CustomS3TemplatesDirName = "s3"
|
||||
CustomGitHubTemplatesDirName = "github"
|
||||
|
|
|
@ -4,6 +4,8 @@ import (
|
|||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
)
|
||||
|
||||
// DiskCatalog is a template catalog helper implementation based on disk
|
||||
|
@ -19,7 +21,7 @@ func NewCatalog(directory string) *DiskCatalog {
|
|||
if directory != "" {
|
||||
catalog.templatesFS = os.DirFS(directory)
|
||||
} else {
|
||||
catalog.templatesFS = os.DirFS("./")
|
||||
catalog.templatesFS = os.DirFS(config.DefaultConfig.GetTemplateDir())
|
||||
}
|
||||
return catalog
|
||||
}
|
||||
|
|
|
@ -50,7 +50,7 @@ var errNoValidCombination = errors.New("no valid combination found")
|
|||
|
||||
// tryResolve attempts to load locate the target by iterating across all the folders tree
|
||||
func (c *DiskCatalog) tryResolve(fullPath string) (string, error) {
|
||||
if fileutil.FileExists(fullPath) {
|
||||
if fileutil.FileOrFolderExists(fullPath) {
|
||||
return fullPath, nil
|
||||
}
|
||||
return "", errNoValidCombination
|
||||
|
|
|
@ -159,6 +159,7 @@ func New(config *Config) (*Store, error) {
|
|||
if _, err := urlutil.Parse(v); err == nil {
|
||||
remoteTemplates = append(remoteTemplates, handleTemplatesEditorURLs(v))
|
||||
} else {
|
||||
|
||||
templatesFinal = append(templatesFinal, v) // something went wrong, treat it as a file
|
||||
}
|
||||
}
|
||||
|
@ -187,6 +188,7 @@ func New(config *Config) (*Store, error) {
|
|||
if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 && !urlBasedTemplatesProvided {
|
||||
store.finalTemplates = []string{cfg.DefaultConfig.TemplatesDirectory}
|
||||
}
|
||||
|
||||
return store, nil
|
||||
}
|
||||
|
||||
|
|
|
@ -65,6 +65,8 @@ func NewStatsTicker(duration int, active, outputJSON, cloud bool, port int) (Pro
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// only print in verbose mode
|
||||
gologger.Verbose().Msgf("Started metrics server at localhost:%v", stats.Options.ListenPort)
|
||||
progress.cloud = cloud
|
||||
progress.active = active
|
||||
progress.stats = stats
|
||||
|
|
|
@ -28,16 +28,6 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st
|
|||
for name, payload := range payloads {
|
||||
payloadsFinal[name] = payload
|
||||
}
|
||||
for name, payload := range payloads {
|
||||
payloadStr, ok := payload.(string)
|
||||
if ok {
|
||||
final, resolveErr := catalog.ResolvePath(payloadStr, templatePath)
|
||||
if resolveErr != nil {
|
||||
return nil, errors.Wrap(resolveErr, "could not read payload file")
|
||||
}
|
||||
payloadsFinal[name] = final
|
||||
}
|
||||
}
|
||||
|
||||
generator := &PayloadGenerator{catalog: catalog, options: opts}
|
||||
if err := generator.validate(payloadsFinal, templatePath); err != nil {
|
||||
|
|
|
@ -6,6 +6,7 @@ import (
|
|||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
folderutil "github.com/projectdiscovery/utils/folder"
|
||||
|
@ -25,7 +26,20 @@ func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePat
|
|||
if fileutil.FileExists(payloadType) {
|
||||
continue
|
||||
}
|
||||
// if file already exists in nuclei-templates directory, skip any further checks
|
||||
if fileutil.FileExists(filepath.Join(config.DefaultConfig.GetTemplateDir(), payloadType)) {
|
||||
continue
|
||||
}
|
||||
|
||||
// in below code, we calculate all possible paths from root and try to resolve the payload
|
||||
// at each level of the path. if the payload is found, we break the loop and continue
|
||||
// ex: template-path: /home/user/nuclei-templates/cves/2020/CVE-2020-1234.yaml
|
||||
// then we check if helper file "my-payload.txt" exists at below paths:
|
||||
// 1. /home/user/nuclei-templates/cves/2020/my-payload.txt
|
||||
// 2. /home/user/nuclei-templates/cves/my-payload.txt
|
||||
// 3. /home/user/nuclei-templates/my-payload.txt
|
||||
// 4. /home/user/my-payload.txt
|
||||
// 5. /home/my-payload.txt
|
||||
changed := false
|
||||
|
||||
dir, _ := filepath.Split(templatePath)
|
||||
|
|
Loading…
Reference in New Issue