mirror of https://github.com/daffainfo/nuclei.git
SDK: abstracted and minimal nuclei v3 sdk (#4104)
* new sdk progress * nuclei v3 new sdk/library * fix TestActionGetResource broken link * fix clistats + clustering and more * fix lint error * fix missing ticker * update advanced library usage example * fix integration tests * misc update * add utm_source and fix lint error --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>dev
Tarun Koyalwar
GitHub
parent
f7fe99f806
commit
2d317884b5
|
|
@ -56,6 +56,10 @@ jobs:
|
|||
run: go run -race . -l ../functional-test/targets.txt -id tech-detect,tls-version
|
||||
working-directory: v2/cmd/nuclei/
|
||||
|
||||
- name: Example Code Tests
|
||||
run: go build .
|
||||
working-directory: v2/examples/
|
||||
- name: Example SDK Simple
|
||||
run: go run .
|
||||
working-directory: v2/examples/simple/
|
||||
|
||||
- name: Example SDK Advanced
|
||||
run: go run .
|
||||
working-directory: v2/examples/advanced/
|
||||
|
|
|
|||
|
|
@ -379,7 +379,7 @@ We have [a discussion thread around this](https://github.com/projectdiscovery/nu
|
|||
|
||||
### Using Nuclei From Go Code
|
||||
|
||||
Examples of using Nuclei From Go Code to run templates on targets are provided in the [examples](v2/examples/) folder.
|
||||
Complete guide of using Nuclei as Library/SDK is available at [lib](v2/lib/README.md)
|
||||
|
||||
|
||||
### Resources
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ var templatesPathTestCases = []TestCaseInfo{
|
|||
//template folder path issue
|
||||
{Path: "protocols/http/get.yaml", TestCase: &folderPathTemplateTest{}},
|
||||
//cwd
|
||||
{Path: "./protocols/dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}},
|
||||
{Path: "./dns/detect-dangling-cname.yaml", TestCase: &cwdTemplateTest{}},
|
||||
//relative path
|
||||
{Path: "dns/dns-saas-service-detection.yaml", TestCase: &relativePathTemplateTest{}},
|
||||
//absolute path
|
||||
|
|
|
|||
|
|
@ -361,7 +361,6 @@ on extensive configurability, massive extensibility and ease of use.`)
|
|||
flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"),
|
||||
flagSet.BoolVarP(&options.StatsJSON, "stats-json", "sj", false, "display statistics in JSONL(ines) format"),
|
||||
flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"),
|
||||
flagSet.BoolVarP(&options.Metrics, "metrics", "m", false, "expose nuclei metrics on a port"),
|
||||
flagSet.IntVarP(&options.MetricsPort, "metrics-port", "mp", 9092, "port to expose nuclei metrics on"),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
nuclei "github.com/projectdiscovery/nuclei/v2/lib"
|
||||
"github.com/remeh/sizedwaitgroup"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// create nuclei engine with options
|
||||
ne, err := nuclei.NewThreadSafeNucleiEngine()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// setup sizedWaitgroup to handle concurrency
|
||||
sg := sizedwaitgroup.New(10)
|
||||
|
||||
// scan 1 = run dns templates on scanme.sh
|
||||
sg.Add()
|
||||
go func() {
|
||||
defer sg.Done()
|
||||
err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"},
|
||||
nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}),
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// scan 2 = run templates with oast tags on honey.scanme.sh
|
||||
sg.Add()
|
||||
go func() {
|
||||
defer sg.Done()
|
||||
err = ne.ExecuteNucleiWithOpts([]string{"http://honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"oast"}}))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// wait for all scans to finish
|
||||
sg.Wait()
|
||||
defer ne.Close()
|
||||
|
||||
// Output:
|
||||
// [dns-saas-service-detection] scanme.sh
|
||||
// [nameserver-fingerprint] scanme.sh
|
||||
// [dns-saas-service-detection] honey.scanme.sh
|
||||
}
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/logrusorgru/aurora"
|
||||
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/projectdiscovery/httpx/common/httpx"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/core"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cache := hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount, nil)
|
||||
defer cache.Close()
|
||||
|
||||
mockProgress := &testutils.MockProgressClient{}
|
||||
reportingClient, _ := reporting.New(&reporting.Options{}, "")
|
||||
defer reportingClient.Close()
|
||||
|
||||
outputWriter := testutils.NewMockOutputWriter()
|
||||
outputWriter.WriteCallback = func(event *output.ResultEvent) {
|
||||
fmt.Printf("Got Result: %v\n", event)
|
||||
}
|
||||
|
||||
defaultOpts := types.DefaultOptions()
|
||||
protocolstate.Init(defaultOpts)
|
||||
protocolinit.Init(defaultOpts)
|
||||
|
||||
defaultOpts.IncludeIds = goflags.StringSlice{"cname-service", "tech-detect"}
|
||||
defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags
|
||||
|
||||
interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress)
|
||||
interactClient, err := interactsh.New(interactOpts)
|
||||
if err != nil {
|
||||
log.Fatalf("Could not create interact client: %s\n", err)
|
||||
}
|
||||
defer interactClient.Close()
|
||||
|
||||
home, _ := os.UserHomeDir()
|
||||
catalog := disk.NewCatalog(filepath.Join(home, "nuclei-templates"))
|
||||
executerOpts := protocols.ExecutorOptions{
|
||||
Output: outputWriter,
|
||||
Options: defaultOpts,
|
||||
Progress: mockProgress,
|
||||
Catalog: catalog,
|
||||
IssuesClient: reportingClient,
|
||||
RateLimiter: ratelimit.New(context.Background(), 150, time.Second),
|
||||
Interactsh: interactClient,
|
||||
HostErrorsCache: cache,
|
||||
Colorizer: aurora.NewAurora(true),
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
}
|
||||
engine := core.New(defaultOpts)
|
||||
engine.SetExecuterOptions(executerOpts)
|
||||
|
||||
workflowLoader, err := parsers.NewLoader(&executerOpts)
|
||||
if err != nil {
|
||||
log.Fatalf("Could not create workflow loader: %s\n", err)
|
||||
}
|
||||
executerOpts.WorkflowLoader = workflowLoader
|
||||
|
||||
store, err := loader.New(loader.NewConfig(defaultOpts, catalog, executerOpts))
|
||||
if err != nil {
|
||||
log.Fatalf("Could not create loader client: %s\n", err)
|
||||
}
|
||||
store.Load()
|
||||
|
||||
// flat input without probe
|
||||
inputArgs := []*contextargs.MetaInput{{Input: "docs.hackerone.com"}}
|
||||
input := &inputs.SimpleInputProvider{Inputs: inputArgs}
|
||||
|
||||
httpxOptions := httpx.DefaultOptions
|
||||
httpxOptions.Timeout = 5 * time.Second
|
||||
httpxClient, err := httpx.New(&httpxOptions)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// use httpx to probe the URL => https://scanme.sh
|
||||
input.SetWithProbe("scanme.sh", httpxClient)
|
||||
|
||||
_ = engine.Execute(store.Templates(), input)
|
||||
engine.WorkPool().Wait() // Wait for the scan to finish
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
package main
|
||||
|
||||
import nuclei "github.com/projectdiscovery/nuclei/v2/lib"
|
||||
|
||||
func main() {
|
||||
ne, err := nuclei.NewNucleiEngine(
|
||||
nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"oast"}}),
|
||||
nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}), // optionally enable metrics server for better observability
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// load targets and optionally probe non http/https targets
|
||||
ne.LoadTargets([]string{"http://honey.scanme.sh"}, false)
|
||||
err = ne.ExecuteWithCallback(nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer ne.Close()
|
||||
}
|
||||
|
|
@ -103,3 +103,29 @@ func isEmptyDir(dir string) bool {
|
|||
})
|
||||
return !hasFiles
|
||||
}
|
||||
|
||||
// getUtmSource returns utm_source from environment variable
|
||||
func getUtmSource() string {
|
||||
value := ""
|
||||
switch {
|
||||
case os.Getenv("GH_ACTION") != "":
|
||||
value = "ghci"
|
||||
case os.Getenv("TRAVIS") != "":
|
||||
value = "travis"
|
||||
case os.Getenv("CIRCLECI") != "":
|
||||
value = "circleci"
|
||||
case os.Getenv("CI") != "":
|
||||
value = "gitlabci" // this also includes bitbucket
|
||||
case os.Getenv("GITHUB_ACTIONS") != "":
|
||||
value = "ghci"
|
||||
case os.Getenv("AWS_EXECUTION_ENV") != "":
|
||||
value = os.Getenv("AWS_EXECUTION_ENV")
|
||||
case os.Getenv("JENKINS_URL") != "":
|
||||
value = "jenkins"
|
||||
case os.Getenv("FUNCTION_TARGET") != "":
|
||||
value = "gcf"
|
||||
default:
|
||||
value = "unknown"
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"net/url"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
|
|
@ -34,7 +35,56 @@ type PdtmAPIResponse struct {
|
|||
// and returns an error if it fails to check on success it returns nil and changes are
|
||||
// made to the default config in config.DefaultConfig
|
||||
func NucleiVersionCheck() error {
|
||||
resp, err := retryableHttpClient.Get(pdtmNucleiVersionEndpoint + "?" + getpdtmParams())
|
||||
return doVersionCheck(false)
|
||||
}
|
||||
|
||||
// this will be updated by features of 1.21 release (which directly provides sync.Once(func()))
|
||||
type sdkUpdateCheck struct {
|
||||
sync.Once
|
||||
}
|
||||
|
||||
var sdkUpdateCheckInstance = &sdkUpdateCheck{}
|
||||
|
||||
// NucleiSDKVersionCheck checks for latest version of nuclei which running in sdk mode
|
||||
// this only happens once per process regardless of how many times this function is called
|
||||
func NucleiSDKVersionCheck() {
|
||||
sdkUpdateCheckInstance.Do(func() {
|
||||
_ = doVersionCheck(true)
|
||||
})
|
||||
}
|
||||
|
||||
// getpdtmParams returns encoded query parameters sent to update check endpoint
|
||||
func getpdtmParams(isSDK bool) string {
|
||||
params := &url.Values{}
|
||||
params.Add("os", runtime.GOOS)
|
||||
params.Add("arch", runtime.GOARCH)
|
||||
params.Add("go_version", runtime.Version())
|
||||
params.Add("v", config.Version)
|
||||
if isSDK {
|
||||
params.Add("sdk", "true")
|
||||
}
|
||||
params.Add("utm_source", getUtmSource())
|
||||
return params.Encode()
|
||||
}
|
||||
|
||||
// UpdateIgnoreFile updates default ignore file by downloading latest ignore file
|
||||
func UpdateIgnoreFile() error {
|
||||
resp, err := retryableHttpClient.Get(pdtmNucleiIgnoreFileEndpoint + "?" + getpdtmParams(false))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bin, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(config.DefaultConfig.GetIgnoreFilePath(), bin, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
return config.DefaultConfig.UpdateNucleiIgnoreHash()
|
||||
}
|
||||
|
||||
func doVersionCheck(isSDK bool) error {
|
||||
resp, err := retryableHttpClient.Get(pdtmNucleiVersionEndpoint + "?" + getpdtmParams(isSDK))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -63,29 +113,3 @@ func NucleiVersionCheck() error {
|
|||
}
|
||||
return config.DefaultConfig.WriteVersionCheckData(pdtmResp.IgnoreHash, nucleiversion, templateversion)
|
||||
}
|
||||
|
||||
// getpdtmParams returns encoded query parameters sent to update check endpoint
|
||||
func getpdtmParams() string {
|
||||
params := &url.Values{}
|
||||
params.Add("os", runtime.GOOS)
|
||||
params.Add("arch", runtime.GOARCH)
|
||||
params.Add("go_version", runtime.Version())
|
||||
params.Add("v", config.Version)
|
||||
return params.Encode()
|
||||
}
|
||||
|
||||
// UpdateIgnoreFile updates default ignore file by downloading latest ignore file
|
||||
func UpdateIgnoreFile() error {
|
||||
resp, err := retryableHttpClient.Get(pdtmNucleiIgnoreFileEndpoint + "?" + getpdtmParams())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bin, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(config.DefaultConfig.GetIgnoreFilePath(), bin, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
return config.DefaultConfig.UpdateNucleiIgnoreHash()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ func ParseOptions(options *types.Options) {
|
|||
}
|
||||
// Validate the options passed by the user and if any
|
||||
// invalid options have been used, exit.
|
||||
if err := validateOptions(options); err != nil {
|
||||
if err := ValidateOptions(options); err != nil {
|
||||
gologger.Fatal().Msgf("Program exiting: %s\n", err)
|
||||
}
|
||||
|
||||
|
|
@ -105,7 +105,7 @@ func ParseOptions(options *types.Options) {
|
|||
}
|
||||
|
||||
// validateOptions validates the configuration options passed
|
||||
func validateOptions(options *types.Options) error {
|
||||
func ValidateOptions(options *types.Options) error {
|
||||
validate := validator.New()
|
||||
if err := validate.Struct(options); err != nil {
|
||||
if _, ok := err.(*validator.InvalidValidationError); ok {
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ func New(options *types.Options) (*Runner, error) {
|
|||
statsInterval = -1
|
||||
options.EnableProgressBar = true
|
||||
}
|
||||
runner.progress, progressErr = progress.NewStatsTicker(statsInterval, options.EnableProgressBar, options.StatsJSON, options.Metrics, options.Cloud, options.MetricsPort)
|
||||
runner.progress, progressErr = progress.NewStatsTicker(statsInterval, options.EnableProgressBar, options.StatsJSON, options.Cloud, options.MetricsPort)
|
||||
if progressErr != nil {
|
||||
return nil, progressErr
|
||||
}
|
||||
|
|
@ -662,16 +662,6 @@ func (r *Runner) executeSmartWorkflowInput(executorOpts protocols.ExecutorOption
|
|||
}
|
||||
|
||||
func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine) (*atomic.Bool, error) {
|
||||
var unclusteredRequests int64
|
||||
for _, template := range store.Templates() {
|
||||
// workflows will dynamically adjust the totals while running, as
|
||||
// it can't be known in advance which requests will be called
|
||||
if len(template.Workflows) > 0 {
|
||||
continue
|
||||
}
|
||||
unclusteredRequests += int64(template.TotalRequests) * r.hmapInputProvider.Count()
|
||||
}
|
||||
|
||||
if r.options.VerboseVerbose {
|
||||
for _, template := range store.Templates() {
|
||||
r.logAvailableTemplate(template.Path)
|
||||
|
|
@ -681,34 +671,15 @@ func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine)
|
|||
}
|
||||
}
|
||||
|
||||
// Cluster the templates first because we want info on how many
|
||||
// templates did we cluster for showing to user in CLI
|
||||
originalTemplatesCount := len(store.Templates())
|
||||
finalTemplates, clusterCount := templates.ClusterTemplates(store.Templates(), engine.ExecuterOptions())
|
||||
finalTemplates := []*templates.Template{}
|
||||
finalTemplates = append(finalTemplates, store.Templates()...)
|
||||
finalTemplates = append(finalTemplates, store.Workflows()...)
|
||||
|
||||
var totalRequests int64
|
||||
for _, t := range finalTemplates {
|
||||
if len(t.Workflows) > 0 {
|
||||
continue
|
||||
}
|
||||
totalRequests += int64(t.Executer.Requests()) * r.hmapInputProvider.Count()
|
||||
}
|
||||
if totalRequests < unclusteredRequests {
|
||||
gologger.Info().Msgf("Templates clustered: %d (Reduced %d Requests)", clusterCount, unclusteredRequests-totalRequests)
|
||||
}
|
||||
workflowCount := len(store.Workflows())
|
||||
templateCount := originalTemplatesCount + workflowCount
|
||||
|
||||
// 0 matches means no templates were found in the directory
|
||||
if templateCount == 0 {
|
||||
return &atomic.Bool{}, errors.New("no valid templates were found")
|
||||
if len(finalTemplates) == 0 {
|
||||
return nil, errors.New("no templates provided for scan")
|
||||
}
|
||||
|
||||
// tracks global progress and captures stdout/stderr until p.Wait finishes
|
||||
r.progress.Init(r.hmapInputProvider.Count(), templateCount, totalRequests)
|
||||
|
||||
results := engine.ExecuteScanWithOpts(finalTemplates, r.hmapInputProvider, true)
|
||||
results := engine.ExecuteScanWithOpts(finalTemplates, r.hmapInputProvider, r.options.DisableClustering)
|
||||
return results, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,87 @@
|
|||
## Using Nuclei as Library
|
||||
|
||||
Nuclei was primarily built as a CLI tool, but with increasing choice of users wanting to use nuclei as library in their own automation, we have added a simplified Library/SDK of nuclei in v3
|
||||
|
||||
### Installation
|
||||
|
||||
To add nuclei as a library to your go project, you can use the following command:
|
||||
|
||||
```bash
|
||||
go get -u github.com/projectdiscovery/nuclei/v2/lib
|
||||
```
|
||||
|
||||
Or add below import to your go file and let IDE handle the rest:
|
||||
|
||||
```go
|
||||
import nuclei "github.com/projectdiscovery/nuclei/v2/lib"
|
||||
```
|
||||
|
||||
## Basic Example of using Nuclei Library/SDK
|
||||
|
||||
```go
|
||||
// create nuclei engine with options
|
||||
ne, err := nuclei.NewNucleiEngine(
|
||||
nuclei.WithTemplateFilters(nuclei.TemplateFilters{Severity: "critical"}), // run critical severity templates only
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// load targets and optionally probe non http/https targets
|
||||
ne.LoadTargets([]string{"scanme.sh"}, false)
|
||||
err = ne.ExecuteWithCallback(nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer ne.Close()
|
||||
```
|
||||
|
||||
## Advanced Example of using Nuclei Library/SDK
|
||||
|
||||
For Various use cases like batching etc you might want to run nuclei in goroutines this can be done by using `nuclei.NewThreadSafeNucleiEngine`
|
||||
|
||||
```go
|
||||
// create nuclei engine with options
|
||||
ne, err := nuclei.NewThreadSafeNucleiEngine()
|
||||
if err != nil{
|
||||
panic(err)
|
||||
}
|
||||
// setup waitgroup to handle concurrency
|
||||
wg := &sync.WaitGroup{}
|
||||
|
||||
// scan 1 = run dns templates on scanme.sh
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "http"}))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// scan 2 = run http templates on honey.scanme.sh
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// wait for all scans to finish
|
||||
wg.Wait()
|
||||
defer ne.Close()
|
||||
```
|
||||
|
||||
## More Documentation
|
||||
|
||||
For complete documentation of nuclei library, please refer to [godoc](https://pkg.go.dev/github.com/projectdiscovery/nuclei/v2/lib) which contains all available options and methods.
|
||||
|
||||
|
||||
|
||||
### Note
|
||||
|
||||
| :exclamation: **Disclaimer** |
|
||||
|---------------------------------|
|
||||
| **This project is in active development**. Expect breaking changes with releases. Review the release changelog before updating. |
|
||||
| This project was primarily built to be used as a standalone CLI tool. **Running nuclei as a service may pose security risks.** It's recommended to use with caution and additional security measures. |
|
||||
|
|
@ -0,0 +1,298 @@
|
|||
package nuclei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
)
|
||||
|
||||
// config contains all SDK configuration options
|
||||
type TemplateFilters struct {
|
||||
Severity string // filter by severities (accepts CSV values of info, low, medium, high, critical)
|
||||
ExcludeSeverities string // filter by excluding severities (accepts CSV values of info, low, medium, high, critical)
|
||||
ProtocolTypes string // filter by protocol types
|
||||
ExcludeProtocolTypes string // filter by excluding protocol types
|
||||
Authors []string // fiter by author
|
||||
Tags []string // filter by tags present in template
|
||||
ExcludeTags []string // filter by excluding tags present in template
|
||||
IncludeTags []string // filter by including tags present in template
|
||||
IDs []string // filter by template IDs
|
||||
ExcludeIDs []string // filter by excluding template IDs
|
||||
TemplateCondition []string // DSL condition/ expression
|
||||
}
|
||||
|
||||
// WithTemplateFilters sets template filters and only templates matching the filters will be
|
||||
// loaded and executed
|
||||
func WithTemplateFilters(filters TemplateFilters) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
s := severity.Severities{}
|
||||
if err := s.Set(filters.Severity); err != nil {
|
||||
return err
|
||||
}
|
||||
es := severity.Severities{}
|
||||
if err := es.Set(filters.ExcludeSeverities); err != nil {
|
||||
return err
|
||||
}
|
||||
pt := types.ProtocolTypes{}
|
||||
if err := pt.Set(filters.ProtocolTypes); err != nil {
|
||||
return err
|
||||
}
|
||||
ept := types.ProtocolTypes{}
|
||||
if err := ept.Set(filters.ExcludeProtocolTypes); err != nil {
|
||||
return err
|
||||
}
|
||||
e.opts.Authors = filters.Authors
|
||||
e.opts.Tags = filters.Tags
|
||||
e.opts.ExcludeTags = filters.ExcludeTags
|
||||
e.opts.IncludeTags = filters.IncludeTags
|
||||
e.opts.IncludeIds = filters.IDs
|
||||
e.opts.ExcludeIds = filters.ExcludeIDs
|
||||
e.opts.Severities = s
|
||||
e.opts.ExcludeSeverities = es
|
||||
e.opts.Protocols = pt
|
||||
e.opts.ExcludeProtocols = ept
|
||||
e.opts.IncludeConditions = filters.TemplateCondition
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// InteractshOpts contains options for interactsh
|
||||
type InteractshOpts interactsh.Options
|
||||
|
||||
// WithInteractshOptions sets interactsh options
|
||||
func WithInteractshOptions(opts InteractshOpts) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
if e.mode == threadSafe {
|
||||
return ErrOptionsNotSupported.Msgf("WithInteractshOptions")
|
||||
}
|
||||
optsPtr := &opts
|
||||
e.interactshOpts = (*interactsh.Options)(optsPtr)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Concurrency options
|
||||
type Concurrency struct {
|
||||
TemplateConcurrency int // number of templates to run concurrently (per host in host-spray mode)
|
||||
HostConcurrency int // number of hosts to scan concurrently (per template in template-spray mode)
|
||||
HeadlessHostConcurrency int // number of hosts to scan concurrently for headless templates (per template in template-spray mode)
|
||||
HeadlessTemplateConcurrency int // number of templates to run concurrently for headless templates (per host in host-spray mode)
|
||||
}
|
||||
|
||||
// WithConcurrency sets concurrency options
|
||||
func WithConcurrency(opts Concurrency) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
e.opts.TemplateThreads = opts.TemplateConcurrency
|
||||
e.opts.BulkSize = opts.HostConcurrency
|
||||
e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency
|
||||
e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithGlobalRateLimit sets global rate (i.e all hosts combined) limit options
|
||||
func WithGlobalRateLimit(maxTokens int, duration time.Duration) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
e.rateLimiter = ratelimit.New(context.Background(), uint(maxTokens), duration)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// HeadlessOpts contains options for headless templates
|
||||
type HeadlessOpts struct {
|
||||
PageTimeout int // timeout for page load
|
||||
ShowBrowser bool
|
||||
HeadlessOptions []string
|
||||
UseChrome bool
|
||||
}
|
||||
|
||||
// EnableHeadless allows execution of headless templates
|
||||
// *Use With Caution*: Enabling headless mode may open up attack surface due to browser usage
|
||||
// and can be prone to exploitation by custom unverified templates if not properly configured
|
||||
func EnableHeadlessWithOpts(hopts *HeadlessOpts) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
e.opts.Headless = true
|
||||
if hopts != nil {
|
||||
e.opts.HeadlessOptionalArguments = hopts.HeadlessOptions
|
||||
e.opts.PageTimeout = hopts.PageTimeout
|
||||
e.opts.ShowBrowser = hopts.ShowBrowser
|
||||
e.opts.UseInstalledChrome = hopts.UseChrome
|
||||
}
|
||||
if engine.MustDisableSandbox() {
|
||||
gologger.Warning().Msgf("The current platform and privileged user will run the browser without sandbox\n")
|
||||
}
|
||||
browser, err := engine.New(e.opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
e.executerOpts.Browser = browser
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// StatsOptions
|
||||
type StatsOptions struct {
|
||||
Interval int
|
||||
JSON bool
|
||||
MetricServerPort int
|
||||
}
|
||||
|
||||
// EnableStats enables Stats collection with defined interval(in sec) and callback
|
||||
// Note: callback is executed in a separate goroutine
|
||||
func EnableStatsWithOpts(opts StatsOptions) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
if e.mode == threadSafe {
|
||||
return ErrOptionsNotSupported.Msgf("EnableStatsWithOpts")
|
||||
}
|
||||
if opts.Interval == 0 {
|
||||
opts.Interval = 5 //sec
|
||||
}
|
||||
e.opts.StatsInterval = opts.Interval
|
||||
e.enableStats = true
|
||||
e.opts.StatsJSON = opts.JSON
|
||||
e.opts.MetricsPort = opts.MetricServerPort
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// VerbosityOptions
|
||||
type VerbosityOptions struct {
|
||||
Verbose bool // show verbose output
|
||||
Silent bool // show only results
|
||||
Debug bool // show debug output
|
||||
DebugRequest bool // show request in debug output
|
||||
DebugResponse bool // show response in debug output
|
||||
ShowVarDump bool // show variable dumps in output
|
||||
}
|
||||
|
||||
// WithVerbosity allows setting verbosity options of (internal) nuclei engine
|
||||
// and does not affect SDK output
|
||||
func WithVerbosity(opts VerbosityOptions) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
if e.mode == threadSafe {
|
||||
return ErrOptionsNotSupported.Msgf("WithVerbosity")
|
||||
}
|
||||
e.opts.Verbose = opts.Verbose
|
||||
e.opts.Silent = opts.Silent
|
||||
e.opts.Debug = opts.Debug
|
||||
e.opts.DebugRequests = opts.DebugRequest
|
||||
e.opts.DebugResponse = opts.DebugResponse
|
||||
if opts.ShowVarDump {
|
||||
vardump.EnableVarDump = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NetworkConfig contains network config options
|
||||
// ex: retries , httpx probe , timeout etc
|
||||
type NetworkConfig struct {
|
||||
Timeout int // Timeout in seconds
|
||||
Retries int // Number of retries
|
||||
LeaveDefaultPorts bool // Leave default ports for http/https
|
||||
MaxHostError int // Maximum number of host errors to allow before skipping that host
|
||||
TrackError []string // Adds given errors to max host error watchlist
|
||||
DisableMaxHostErr bool // Disable max host error optimization (Hosts are not skipped even if they are not responding)
|
||||
}
|
||||
|
||||
// WithNetworkConfig allows setting network config options
|
||||
func WithNetworkConfig(opts NetworkConfig) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
if e.mode == threadSafe {
|
||||
return ErrOptionsNotSupported.Msgf("WithNetworkConfig")
|
||||
}
|
||||
e.opts.Timeout = opts.Timeout
|
||||
e.opts.Retries = opts.Retries
|
||||
e.opts.LeaveDefaultPorts = opts.LeaveDefaultPorts
|
||||
e.hostErrCache = hosterrorscache.New(opts.MaxHostError, hosterrorscache.DefaultMaxHostsCount, opts.TrackError)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithProxy allows setting proxy options
|
||||
func WithProxy(proxy []string, proxyInternalRequests bool) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
if e.mode == threadSafe {
|
||||
return ErrOptionsNotSupported.Msgf("WithProxy")
|
||||
}
|
||||
e.opts.Proxy = proxy
|
||||
e.opts.ProxyInternal = proxyInternalRequests
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithScanStrategy allows setting scan strategy options
|
||||
func WithScanStrategy(strategy string) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
e.opts.ScanStrategy = strategy
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// OutputWriter
|
||||
type OutputWriter output.Writer
|
||||
|
||||
// UseWriter allows setting custom output writer
|
||||
// by default a mock writer is used with user defined callback
|
||||
// if outputWriter is used callback will be ignored
|
||||
func UseOutputWriter(writer OutputWriter) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
if e.mode == threadSafe {
|
||||
return ErrOptionsNotSupported.Msgf("UseOutputWriter")
|
||||
}
|
||||
e.customWriter = writer
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// StatsWriter
|
||||
type StatsWriter progress.Progress
|
||||
|
||||
// UseStatsWriter allows setting a custom stats writer
|
||||
// which can be used to write stats somewhere (ex: send to webserver etc)
|
||||
func UseStatsWriter(writer StatsWriter) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
if e.mode == threadSafe {
|
||||
return ErrOptionsNotSupported.Msgf("UseStatsWriter")
|
||||
}
|
||||
e.customProgress = writer
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithTemplateUpdateCallback allows setting a callback which will be called
|
||||
// when nuclei templates are outdated
|
||||
// Note: Nuclei-templates are crucial part of nuclei and using outdated templates or nuclei sdk is not recommended
|
||||
// as it may cause unexpected results due to compatibility issues
|
||||
func WithTemplateUpdateCallback(disableTemplatesAutoUpgrade bool, callback func(newVersion string)) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
if e.mode == threadSafe {
|
||||
return ErrOptionsNotSupported.Msgf("WithTemplateUpdateCallback")
|
||||
}
|
||||
e.disableTemplatesAutoUpgrade = disableTemplatesAutoUpgrade
|
||||
e.onUpdateAvailableCallback = callback
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithSandboxOptions allows setting supported sandbox options
|
||||
func WithSandboxOptions(allowLocalFileAccess bool, restrictLocalNetworkAccess bool) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
if e.mode == threadSafe {
|
||||
return ErrOptionsNotSupported.Msgf("WithSandboxOptions")
|
||||
}
|
||||
e.opts.AllowLocalFileAccess = allowLocalFileAccess
|
||||
e.opts.RestrictLocalNetworkAccess = restrictLocalNetworkAccess
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
//go:build !race
|
||||
// +build !race
|
||||
|
||||
package nuclei_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
nuclei "github.com/projectdiscovery/nuclei/v2/lib"
|
||||
"github.com/remeh/sizedwaitgroup"
|
||||
)
|
||||
|
||||
// A very simple example on how to use nuclei engine
|
||||
func ExampleNucleiEngine() {
|
||||
// create nuclei engine with options
|
||||
ne, err := nuclei.NewNucleiEngine(
|
||||
nuclei.WithTemplateFilters(nuclei.TemplateFilters{IDs: []string{"self-signed-ssl"}}), // only run self-signed-ssl template
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// load targets and optionally probe non http/https targets
|
||||
ne.LoadTargets([]string{"scanme.sh"}, false)
|
||||
// when callback is nil it nuclei will print JSON output to stdout
|
||||
err = ne.ExecuteWithCallback(nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer ne.Close()
|
||||
|
||||
// Output:
|
||||
// [self-signed-ssl] scanme.sh:443
|
||||
}
|
||||
|
||||
func ExampleThreadSafeNucleiEngine() {
|
||||
// create nuclei engine with options
|
||||
ne, err := nuclei.NewThreadSafeNucleiEngine()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// setup sizedWaitgroup to handle concurrency
|
||||
// here we are using sizedWaitgroup to limit concurrency to 1
|
||||
sg := sizedwaitgroup.New(1)
|
||||
|
||||
// scan 1 = run dns templates on scanme.sh
|
||||
sg.Add()
|
||||
go func() {
|
||||
defer sg.Done()
|
||||
err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"},
|
||||
nuclei.WithTemplateFilters(nuclei.TemplateFilters{IDs: []string{"nameserver-fingerprint"}}), // only run self-signed-ssl template
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// scan 2 = run dns templates on honey.scanme.sh
|
||||
sg.Add()
|
||||
go func() {
|
||||
defer sg.Done()
|
||||
err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// wait for all scans to finish
|
||||
sg.Wait()
|
||||
defer ne.Close()
|
||||
|
||||
// Output:
|
||||
// [nameserver-fingerprint] scanme.sh
|
||||
// [dns-saas-service-detection] honey.scanme.sh
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// this file only contains testtables examples https://go.dev/blog/examples
|
||||
// and actual functionality test are in sdk_test.go
|
||||
if os.Getenv("GH_ACTION") != "" {
|
||||
// no need to run this test on github actions
|
||||
return
|
||||
}
|
||||
m.Run()
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
package nuclei
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
|
||||
uncoverNuclei "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/uncover"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
||||
"github.com/projectdiscovery/uncover"
|
||||
)
|
||||
|
||||
// helper.go file proxy execution of all nuclei functions that are nested deep inside multiple packages
|
||||
// but are helpful / useful while using nuclei as a library
|
||||
|
||||
// GetTargetsFromUncover returns targets from uncover in given format .
|
||||
// supported formats are any string with [ip,host,port,url] placeholders
|
||||
func GetTargetsFromUncover(ctx context.Context, outputFormat string, opts *uncover.Options) (chan string, error) {
|
||||
return uncoverNuclei.GetTargetsFromUncover(ctx, outputFormat, opts)
|
||||
}
|
||||
|
||||
// GetTargetsFromTemplateMetadata returns all targets by querying engine metadata (ex: fofo-query,shodan-query) etc from given templates .
|
||||
// supported formats are any string with [ip,host,port,url] placeholders
|
||||
func GetTargetsFromTemplateMetadata(ctx context.Context, templates []*templates.Template, outputFormat string, opts *uncover.Options) chan string {
|
||||
return uncoverNuclei.GetUncoverTargetsFromMetadata(ctx, templates, outputFormat, opts)
|
||||
}
|
||||
|
||||
// DefaultConfig is instance of default nuclei configs
|
||||
// any mutations to this config will be reflected in all nuclei instances (saves some config to disk)
|
||||
var DefaultConfig *config.Config
|
||||
|
||||
func init() {
|
||||
DefaultConfig = config.DefaultConfig
|
||||
}
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
package nuclei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/logrusorgru/aurora"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/core"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
errorutil "github.com/projectdiscovery/utils/errors"
|
||||
)
|
||||
|
||||
// unsafeOptions are those nuclei objects/instances/types
|
||||
// that are required to run nuclei engine but are not thread safe
|
||||
// hence they are ephemeral and are created on every ExecuteNucleiWithOpts invocation
|
||||
// in ThreadSafeNucleiEngine
|
||||
type unsafeOptions struct {
|
||||
executerOpts protocols.ExecutorOptions
|
||||
engine *core.Engine
|
||||
}
|
||||
|
||||
// createEphemeralObjects creates ephemeral nuclei objects/instances/types
|
||||
func createEphemeralObjects(base *NucleiEngine, opts *types.Options) (*unsafeOptions, error) {
|
||||
u := &unsafeOptions{}
|
||||
u.executerOpts = protocols.ExecutorOptions{
|
||||
Output: base.customWriter,
|
||||
Options: opts,
|
||||
Progress: base.customProgress,
|
||||
Catalog: base.catalog,
|
||||
IssuesClient: base.rc,
|
||||
RateLimiter: base.rateLimiter,
|
||||
Interactsh: base.interactshClient,
|
||||
HostErrorsCache: base.hostErrCache,
|
||||
Colorizer: aurora.NewAurora(true),
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
}
|
||||
if opts.RateLimitMinute > 0 {
|
||||
u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimitMinute), time.Minute)
|
||||
} else if opts.RateLimit > 0 {
|
||||
u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimit), time.Second)
|
||||
} else {
|
||||
u.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
|
||||
}
|
||||
u.engine = core.New(opts)
|
||||
u.engine.SetExecuterOptions(u.executerOpts)
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// ThreadSafeNucleiEngine is a tweaked version of nuclei.Engine whose methods are thread-safe
|
||||
// and can be used concurrently. Non-thread-safe methods start with Global prefix
|
||||
type ThreadSafeNucleiEngine struct {
|
||||
eng *NucleiEngine
|
||||
}
|
||||
|
||||
// NewThreadSafeNucleiEngine creates a new nuclei engine with given options
|
||||
// whose methods are thread-safe and can be used concurrently
|
||||
// Note: Non-thread-safe methods start with Global prefix
|
||||
func NewThreadSafeNucleiEngine(opts ...NucleiSDKOptions) (*ThreadSafeNucleiEngine, error) {
|
||||
// default options
|
||||
e := &NucleiEngine{
|
||||
opts: types.DefaultOptions(),
|
||||
mode: threadSafe,
|
||||
}
|
||||
for _, option := range opts {
|
||||
if err := option(e); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := e.init(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ThreadSafeNucleiEngine{eng: e}, nil
|
||||
}
|
||||
|
||||
// GlobalLoadAllTemplates loads all templates from nuclei-templates repo
|
||||
// This method will load all templates based on filters given at the time of nuclei engine creation in opts
|
||||
func (e *ThreadSafeNucleiEngine) GlobalLoadAllTemplates() error {
|
||||
return e.eng.LoadAllTemplates()
|
||||
}
|
||||
|
||||
// GlobalResultCallback sets a callback function which will be called for each result
|
||||
func (e *ThreadSafeNucleiEngine) GlobalResultCallback(callback func(event *output.ResultEvent)) {
|
||||
e.eng.resultCallbacks = []func(*output.ResultEvent){callback}
|
||||
}
|
||||
|
||||
// ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found)
|
||||
// This method can be called concurrently and it will use some global resources but can be runned parllely
|
||||
// by invoking this method with different options and targets
|
||||
// Note: Not all options are thread-safe. this method will throw error if you try to use non-thread-safe options
|
||||
func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ...NucleiSDKOptions) error {
|
||||
baseOpts := *e.eng.opts
|
||||
tmpEngine := &NucleiEngine{opts: &baseOpts, mode: threadSafe}
|
||||
for _, option := range opts {
|
||||
if err := option(tmpEngine); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// create ephemeral nuclei objects/instances/types using base nuclei engine
|
||||
unsafeOpts, err := createEphemeralObjects(e.eng, tmpEngine.opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// load templates
|
||||
workflowLoader, err := parsers.NewLoader(&unsafeOpts.executerOpts)
|
||||
if err != nil {
|
||||
return errorutil.New("Could not create workflow loader: %s\n", err)
|
||||
}
|
||||
unsafeOpts.executerOpts.WorkflowLoader = workflowLoader
|
||||
|
||||
store, err := loader.New(loader.NewConfig(tmpEngine.opts, e.eng.catalog, unsafeOpts.executerOpts))
|
||||
if err != nil {
|
||||
return errorutil.New("Could not create loader client: %s\n", err)
|
||||
}
|
||||
store.Load()
|
||||
|
||||
inputProvider := &inputs.SimpleInputProvider{
|
||||
Inputs: []*contextargs.MetaInput{},
|
||||
}
|
||||
|
||||
// load targets
|
||||
for _, target := range targets {
|
||||
inputProvider.Set(target)
|
||||
}
|
||||
|
||||
if len(store.Templates()) == 0 && len(store.Workflows()) == 0 {
|
||||
return ErrNoTemplatesAvailable
|
||||
}
|
||||
if inputProvider.Count() == 0 {
|
||||
return ErrNoTargetsAvailable
|
||||
}
|
||||
|
||||
engine := core.New(tmpEngine.opts)
|
||||
engine.SetExecuterOptions(unsafeOpts.executerOpts)
|
||||
|
||||
_ = engine.ExecuteScanWithOpts(store.Templates(), inputProvider, false)
|
||||
|
||||
engine.WorkPool().Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close all resources used by nuclei engine
|
||||
func (e *ThreadSafeNucleiEngine) Close() {
|
||||
e.eng.Close()
|
||||
}
|
||||
|
|
@ -0,0 +1,180 @@
|
|||
package nuclei
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
|
||||
"github.com/projectdiscovery/httpx/common/httpx"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/core"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
errorutil "github.com/projectdiscovery/utils/errors"
|
||||
)
|
||||
|
||||
// NucleiSDKOptions contains options for nuclei SDK
|
||||
type NucleiSDKOptions func(e *NucleiEngine) error
|
||||
|
||||
var (
|
||||
// ErrNotImplemented is returned when a feature is not implemented
|
||||
ErrNotImplemented = errorutil.New("Not implemented")
|
||||
// ErrNoTemplatesAvailable is returned when no templates are available to execute
|
||||
ErrNoTemplatesAvailable = errorutil.New("No templates available")
|
||||
// ErrNoTargetsAvailable is returned when no targets are available to scan
|
||||
ErrNoTargetsAvailable = errorutil.New("No targets available")
|
||||
// ErrOptionsNotSupported is returned when an option is not supported in thread safe mode
|
||||
ErrOptionsNotSupported = errorutil.NewWithFmt("Option %v not supported in thread safe mode")
|
||||
)
|
||||
|
||||
type engineMode uint
|
||||
|
||||
const (
|
||||
singleInstance engineMode = iota
|
||||
threadSafe
|
||||
)
|
||||
|
||||
// NucleiEngine is the Engine/Client for nuclei which
|
||||
// runs scans using templates and returns results
|
||||
type NucleiEngine struct {
|
||||
// user options
|
||||
resultCallbacks []func(event *output.ResultEvent)
|
||||
onFailureCallback func(event *output.InternalEvent)
|
||||
disableTemplatesAutoUpgrade bool
|
||||
enableStats bool
|
||||
onUpdateAvailableCallback func(newVersion string)
|
||||
|
||||
// ready-status fields
|
||||
templatesLoaded bool
|
||||
|
||||
// unexported core fields
|
||||
interactshClient *interactsh.Client
|
||||
catalog *disk.DiskCatalog
|
||||
rateLimiter *ratelimit.Limiter
|
||||
store *loader.Store
|
||||
httpxClient *httpx.HTTPX
|
||||
inputProvider *inputs.SimpleInputProvider
|
||||
engine *core.Engine
|
||||
mode engineMode
|
||||
browserInstance *engine.Browser
|
||||
httpClient *retryablehttp.Client
|
||||
|
||||
// unexported meta options
|
||||
opts *types.Options
|
||||
interactshOpts *interactsh.Options
|
||||
hostErrCache *hosterrorscache.Cache
|
||||
customWriter output.Writer
|
||||
customProgress progress.Progress
|
||||
rc reporting.Client
|
||||
executerOpts protocols.ExecutorOptions
|
||||
}
|
||||
|
||||
// LoadAllTemplates loads all nuclei template based on given options
|
||||
func (e *NucleiEngine) LoadAllTemplates() error {
|
||||
workflowLoader, err := parsers.NewLoader(&e.executerOpts)
|
||||
if err != nil {
|
||||
return errorutil.New("Could not create workflow loader: %s\n", err)
|
||||
}
|
||||
e.executerOpts.WorkflowLoader = workflowLoader
|
||||
|
||||
e.store, err = loader.New(loader.NewConfig(e.opts, e.catalog, e.executerOpts))
|
||||
if err != nil {
|
||||
return errorutil.New("Could not create loader client: %s\n", err)
|
||||
}
|
||||
e.store.Load()
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetTemplates returns all nuclei templates that are loaded
|
||||
func (e *NucleiEngine) GetTemplates() []*templates.Template {
|
||||
if !e.templatesLoaded {
|
||||
_ = e.LoadAllTemplates()
|
||||
}
|
||||
return e.store.Templates()
|
||||
}
|
||||
|
||||
// LoadTargets(urls/domains/ips only) adds targets to the nuclei engine
|
||||
func (e *NucleiEngine) LoadTargets(targets []string, probeNonHttp bool) {
|
||||
for _, target := range targets {
|
||||
if probeNonHttp {
|
||||
e.inputProvider.SetWithProbe(target, e.httpxClient)
|
||||
} else {
|
||||
e.inputProvider.Set(target)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// LoadTargetsFromReader adds targets(urls/domains/ips only) from reader to the nuclei engine
|
||||
func (e *NucleiEngine) LoadTargetsFromReader(reader io.Reader, probeNonHttp bool) {
|
||||
buff := bufio.NewScanner(reader)
|
||||
for buff.Scan() {
|
||||
if probeNonHttp {
|
||||
e.inputProvider.SetWithProbe(buff.Text(), e.httpxClient)
|
||||
} else {
|
||||
e.inputProvider.Set(buff.Text())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Close all resources used by nuclei engine
|
||||
func (e *NucleiEngine) Close() {
|
||||
e.interactshClient.Close()
|
||||
e.rc.Close()
|
||||
e.customWriter.Close()
|
||||
e.hostErrCache.Close()
|
||||
e.executerOpts.RateLimiter.Stop()
|
||||
}
|
||||
|
||||
// ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found)
|
||||
func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.ResultEvent)) error {
|
||||
if !e.templatesLoaded {
|
||||
_ = e.LoadAllTemplates()
|
||||
}
|
||||
if len(e.store.Templates()) == 0 && len(e.store.Workflows()) == 0 {
|
||||
return ErrNoTemplatesAvailable
|
||||
}
|
||||
if e.inputProvider.Count() == 0 {
|
||||
return ErrNoTargetsAvailable
|
||||
}
|
||||
|
||||
filtered := []func(event *output.ResultEvent){}
|
||||
for _, callback := range callback {
|
||||
if callback != nil {
|
||||
filtered = append(filtered, callback)
|
||||
}
|
||||
}
|
||||
e.resultCallbacks = append(e.resultCallbacks, filtered...)
|
||||
|
||||
_ = e.engine.ExecuteScanWithOpts(e.store.Templates(), e.inputProvider, false)
|
||||
defer e.engine.WorkPool().Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewNucleiEngine creates a new nuclei engine instance
|
||||
func NewNucleiEngine(options ...NucleiSDKOptions) (*NucleiEngine, error) {
|
||||
// default options
|
||||
e := &NucleiEngine{
|
||||
opts: types.DefaultOptions(),
|
||||
mode: singleInstance,
|
||||
}
|
||||
for _, option := range options {
|
||||
if err := option(e); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := e.init(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,197 @@
|
|||
package nuclei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/logrusorgru/aurora"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/gologger/levels"
|
||||
"github.com/projectdiscovery/httpx/common/httpx"
|
||||
"github.com/projectdiscovery/nuclei/v2/internal/installer"
|
||||
"github.com/projectdiscovery/nuclei/v2/internal/runner"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/core"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
)
|
||||
|
||||
// applyRequiredDefaults to options
|
||||
func (e *NucleiEngine) applyRequiredDefaults() {
|
||||
if e.customWriter == nil {
|
||||
mockoutput := testutils.NewMockOutputWriter()
|
||||
mockoutput.WriteCallback = func(event *output.ResultEvent) {
|
||||
if len(e.resultCallbacks) > 0 {
|
||||
for _, callback := range e.resultCallbacks {
|
||||
if callback != nil {
|
||||
callback(event)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
sb := strings.Builder{}
|
||||
sb.WriteString(fmt.Sprintf("[%v] ", event.TemplateID))
|
||||
if event.Matched != "" {
|
||||
sb.WriteString(event.Matched)
|
||||
} else {
|
||||
sb.WriteString(event.Host)
|
||||
}
|
||||
fmt.Println(sb.String())
|
||||
}
|
||||
if e.onFailureCallback != nil {
|
||||
mockoutput.FailureCallback = e.onFailureCallback
|
||||
}
|
||||
e.customWriter = mockoutput
|
||||
}
|
||||
if e.customProgress == nil {
|
||||
e.customProgress = &testutils.MockProgressClient{}
|
||||
}
|
||||
if e.hostErrCache == nil {
|
||||
e.hostErrCache = hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount, nil)
|
||||
}
|
||||
// setup interactsh
|
||||
if e.interactshOpts != nil {
|
||||
e.interactshOpts.Output = e.customWriter
|
||||
e.interactshOpts.Progress = e.customProgress
|
||||
} else {
|
||||
e.interactshOpts = interactsh.DefaultOptions(e.customWriter, e.rc, e.customProgress)
|
||||
}
|
||||
if e.rateLimiter == nil {
|
||||
e.rateLimiter = ratelimit.New(context.Background(), 150, time.Second)
|
||||
}
|
||||
// these templates are known to have weak matchers
|
||||
// and idea is to disable them to avoid false positives
|
||||
e.opts.ExcludeTags = config.ReadIgnoreFile().Tags
|
||||
|
||||
e.inputProvider = &inputs.SimpleInputProvider{
|
||||
Inputs: []*contextargs.MetaInput{},
|
||||
}
|
||||
}
|
||||
|
||||
// init
|
||||
func (e *NucleiEngine) init() error {
|
||||
if e.opts.Verbose {
|
||||
gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
|
||||
} else if e.opts.Debug {
|
||||
gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
|
||||
} else if e.opts.Silent {
|
||||
gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
|
||||
}
|
||||
|
||||
if err := runner.ValidateOptions(e.opts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if e.opts.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" {
|
||||
httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
e.httpClient = httpclient
|
||||
}
|
||||
|
||||
_ = protocolstate.Init(e.opts)
|
||||
_ = protocolinit.Init(e.opts)
|
||||
e.applyRequiredDefaults()
|
||||
var err error
|
||||
|
||||
// setup progressbar
|
||||
if e.enableStats {
|
||||
progressInstance, progressErr := progress.NewStatsTicker(e.opts.StatsInterval, e.enableStats, e.opts.StatsJSON, false, e.opts.MetricsPort)
|
||||
if progressErr != nil {
|
||||
return err
|
||||
}
|
||||
e.customProgress = progressInstance
|
||||
e.interactshOpts.Progress = progressInstance
|
||||
}
|
||||
|
||||
if err := reporting.CreateConfigIfNotExists(); err != nil {
|
||||
return err
|
||||
}
|
||||
// we don't support reporting config in sdk mode
|
||||
if e.rc, err = reporting.New(&reporting.Options{}, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
e.interactshOpts.IssuesClient = e.rc
|
||||
if e.httpClient != nil {
|
||||
e.interactshOpts.HTTPClient = e.httpClient
|
||||
}
|
||||
if e.interactshClient, err = interactsh.New(e.interactshOpts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
e.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory)
|
||||
|
||||
e.executerOpts = protocols.ExecutorOptions{
|
||||
Output: e.customWriter,
|
||||
Options: e.opts,
|
||||
Progress: e.customProgress,
|
||||
Catalog: e.catalog,
|
||||
IssuesClient: e.rc,
|
||||
RateLimiter: e.rateLimiter,
|
||||
Interactsh: e.interactshClient,
|
||||
HostErrorsCache: e.hostErrCache,
|
||||
Colorizer: aurora.NewAurora(true),
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
Browser: e.browserInstance,
|
||||
}
|
||||
|
||||
if e.opts.RateLimitMinute > 0 {
|
||||
e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimitMinute), time.Minute)
|
||||
} else if e.opts.RateLimit > 0 {
|
||||
e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), time.Second)
|
||||
} else {
|
||||
e.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
|
||||
}
|
||||
|
||||
e.engine = core.New(e.opts)
|
||||
e.engine.SetExecuterOptions(e.executerOpts)
|
||||
|
||||
httpxOptions := httpx.DefaultOptions
|
||||
httpxOptions.Timeout = 5 * time.Second
|
||||
if e.httpxClient, err = httpx.New(&httpxOptions); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Only Happens once regardless how many times this function is called
|
||||
// This will update ignore file to filter out templates with weak matchers to avoid false positives
|
||||
// and also upgrade templates to latest version if available
|
||||
installer.NucleiSDKVersionCheck()
|
||||
|
||||
return e.processUpdateCheckResults()
|
||||
}
|
||||
|
||||
type syncOnce struct {
|
||||
sync.Once
|
||||
}
|
||||
|
||||
var updateCheckInstance = &syncOnce{}
|
||||
|
||||
// processUpdateCheckResults processes update check results
|
||||
func (e *NucleiEngine) processUpdateCheckResults() error {
|
||||
var err error
|
||||
updateCheckInstance.Do(func() {
|
||||
if e.onUpdateAvailableCallback != nil {
|
||||
e.onUpdateAvailableCallback(config.DefaultConfig.LatestNucleiTemplatesVersion)
|
||||
}
|
||||
tm := installer.TemplateManager{}
|
||||
err = tm.UpdateIfOutdated()
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
package nuclei_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
nuclei "github.com/projectdiscovery/nuclei/v2/lib"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSimpleNuclei(t *testing.T) {
|
||||
ne, err := nuclei.NewNucleiEngine(
|
||||
nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}),
|
||||
nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}),
|
||||
)
|
||||
require.Nil(t, err)
|
||||
ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here
|
||||
// when callback is nil it nuclei will print JSON output to stdout
|
||||
err = ne.ExecuteWithCallback(nil)
|
||||
require.Nil(t, err)
|
||||
defer ne.Close()
|
||||
}
|
||||
|
||||
func TestThreadSafeNuclei(t *testing.T) {
|
||||
// create nuclei engine with options
|
||||
ne, err := nuclei.NewThreadSafeNucleiEngine()
|
||||
require.Nil(t, err)
|
||||
|
||||
// scan 1 = run dns templates on scanme.sh
|
||||
t.Run("scanme.sh", func(t *testing.T) {
|
||||
err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
|
||||
require.Nil(t, err)
|
||||
})
|
||||
|
||||
// scan 2 = run dns templates on honey.scanme.sh
|
||||
t.Run("honey.scanme.sh", func(t *testing.T) {
|
||||
err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
|
||||
require.Nil(t, err)
|
||||
})
|
||||
|
||||
// wait for all scans to finish
|
||||
defer ne.Close()
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ import (
|
|||
|
||||
"github.com/remeh/sizedwaitgroup"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
||||
|
|
@ -34,13 +35,36 @@ func (e *Engine) ExecuteScanWithOpts(templatesList []*templates.Template, target
|
|||
results := &atomic.Bool{}
|
||||
selfcontainedWg := &sync.WaitGroup{}
|
||||
|
||||
totalReqBeforeCluster := getRequestCount(templatesList) * int(target.Count())
|
||||
|
||||
// attempt to cluster templates if noCluster is false
|
||||
var finalTemplates []*templates.Template
|
||||
clusterCount := 0
|
||||
if !noCluster {
|
||||
finalTemplates, _ = templates.ClusterTemplates(templatesList, e.executerOpts)
|
||||
finalTemplates, clusterCount = templates.ClusterTemplates(templatesList, e.executerOpts)
|
||||
} else {
|
||||
finalTemplates = templatesList
|
||||
}
|
||||
|
||||
totalReqAfterClustering := getRequestCount(finalTemplates) * int(target.Count())
|
||||
|
||||
if !noCluster && totalReqAfterClustering < totalReqBeforeCluster {
|
||||
gologger.Info().Msgf("Templates clustered: %d (Reduced %d Requests)", clusterCount, totalReqBeforeCluster-totalReqAfterClustering)
|
||||
}
|
||||
|
||||
// 0 matches means no templates were found in the directory
|
||||
if len(finalTemplates) == 0 {
|
||||
return &atomic.Bool{}
|
||||
}
|
||||
|
||||
if e.executerOpts.Progress != nil {
|
||||
// Notes:
|
||||
// workflow requests are not counted as they can be conditional
|
||||
// templateList count is user requested templates count (before clustering)
|
||||
// totalReqAfterClustering is total requests count after clustering
|
||||
e.executerOpts.Progress.Init(target.Count(), len(templatesList), int64(totalReqAfterClustering))
|
||||
}
|
||||
|
||||
if stringsutil.EqualFoldAny(e.options.ScanStrategy, scanstrategy.Auto.String(), "") {
|
||||
// TODO: this is only a placeholder, auto scan strategy should choose scan strategy
|
||||
// based on no of hosts , templates , stream and other optimization parameters
|
||||
|
|
@ -122,3 +146,17 @@ func (e *Engine) executeHostSpray(templatesList []*templates.Template, target In
|
|||
wp.Wait()
|
||||
return results
|
||||
}
|
||||
|
||||
// returns total requests count
|
||||
func getRequestCount(templates []*templates.Template) int {
|
||||
count := 0
|
||||
for _, template := range templates {
|
||||
// ignore requests in workflows as total requests in workflow
|
||||
// depends on what templates will be called in workflow
|
||||
if len(template.Workflows) > 0 {
|
||||
continue
|
||||
}
|
||||
count += template.TotalRequests
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import (
|
|||
)
|
||||
|
||||
func TestWorkflowsSimple(t *testing.T) {
|
||||
progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
|
||||
progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
|
||||
|
||||
workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
|
||||
{Executers: []*workflows.ProtocolExecuterPair{{
|
||||
|
|
@ -29,7 +29,7 @@ func TestWorkflowsSimple(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestWorkflowsSimpleMultiple(t *testing.T) {
|
||||
progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
|
||||
progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
|
||||
|
||||
var firstInput, secondInput string
|
||||
workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
|
||||
|
|
@ -54,7 +54,7 @@ func TestWorkflowsSimpleMultiple(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestWorkflowsSubtemplates(t *testing.T) {
|
||||
progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
|
||||
progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
|
||||
|
||||
var firstInput, secondInput string
|
||||
workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
|
||||
|
|
@ -80,7 +80,7 @@ func TestWorkflowsSubtemplates(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
|
||||
progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
|
||||
progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
|
||||
|
||||
var firstInput, secondInput string
|
||||
workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
|
||||
|
|
@ -104,7 +104,7 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
|
||||
progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
|
||||
progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
|
||||
|
||||
var firstInput, secondInput string
|
||||
workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
|
||||
|
|
@ -133,7 +133,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
|
||||
progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
|
||||
progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
|
||||
|
||||
var firstInput, secondInput string
|
||||
workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
|
||||
|
|
|
|||
|
|
@ -4,10 +4,7 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
|
@ -44,13 +41,12 @@ type StatsTicker struct {
|
|||
cloud bool
|
||||
active bool
|
||||
outputJSON bool
|
||||
server *http.Server
|
||||
stats clistats.StatisticsClient
|
||||
tickDuration time.Duration
|
||||
}
|
||||
|
||||
// NewStatsTicker creates and returns a new progress tracking object.
|
||||
func NewStatsTicker(duration int, active, outputJSON, metrics, cloud bool, port int) (Progress, error) {
|
||||
func NewStatsTicker(duration int, active, outputJSON, cloud bool, port int) (Progress, error) {
|
||||
var tickDuration time.Duration
|
||||
if active && duration != -1 {
|
||||
tickDuration = time.Duration(duration) * time.Second
|
||||
|
|
@ -60,7 +56,12 @@ func NewStatsTicker(duration int, active, outputJSON, metrics, cloud bool, port
|
|||
|
||||
progress := &StatsTicker{}
|
||||
|
||||
stats, err := clistats.New()
|
||||
statsOpts := &clistats.DefaultOptions
|
||||
statsOpts.ListenPort = port
|
||||
// metrics port is enabled by default and is not configurable with new version of clistats
|
||||
// by default 63636 is used and than can be modified with -mp flag
|
||||
|
||||
stats, err := clistats.NewWithOptions(context.TODO(), statsOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -70,21 +71,6 @@ func NewStatsTicker(duration int, active, outputJSON, metrics, cloud bool, port
|
|||
progress.tickDuration = tickDuration
|
||||
progress.outputJSON = outputJSON
|
||||
|
||||
if metrics {
|
||||
http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
|
||||
metrics := progress.getMetrics()
|
||||
_ = json.NewEncoder(w).Encode(metrics)
|
||||
})
|
||||
progress.server = &http.Server{
|
||||
Addr: net.JoinHostPort("127.0.0.1", strconv.Itoa(port)),
|
||||
Handler: http.DefaultServeMux,
|
||||
}
|
||||
go func() {
|
||||
if err := progress.server.ListenAndServe(); err != nil {
|
||||
gologger.Warning().Msgf("Could not serve metrics: %s", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
return progress, nil
|
||||
}
|
||||
|
||||
|
|
@ -110,6 +96,7 @@ func (p *StatsTicker) Init(hostCount int64, rulesCount int, requestCount int64)
|
|||
gologger.Warning().Msgf("Couldn't start statistics: %s", err)
|
||||
}
|
||||
|
||||
// Note: this is needed and is responsible for the tick event
|
||||
p.stats.GetStatResponse(p.tickDuration, func(s string, err error) error {
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("Could not read statistics: %s\n", err)
|
||||
|
|
@ -265,11 +252,6 @@ func metricsMap(stats clistats.StatisticsClient) map[string]interface{} {
|
|||
return results
|
||||
}
|
||||
|
||||
// getMetrics returns a map of important metrics for client
|
||||
func (p *StatsTicker) getMetrics() map[string]interface{} {
|
||||
return metricsMap(p.stats)
|
||||
}
|
||||
|
||||
// fmtDuration formats the duration for the time elapsed
|
||||
func fmtDuration(d time.Duration) string {
|
||||
d = d.Round(time.Second)
|
||||
|
|
@ -294,7 +276,4 @@ func (p *StatsTicker) Stop() {
|
|||
gologger.Warning().Msgf("Couldn't stop statistics: %s", err)
|
||||
}
|
||||
}
|
||||
if p.server != nil {
|
||||
_ = p.server.Shutdown(context.Background())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ var executerOpts protocols.ExecutorOptions
|
|||
func setup() {
|
||||
options := testutils.DefaultOptions
|
||||
testutils.Init(options)
|
||||
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
|
||||
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
|
||||
|
||||
executerOpts = protocols.ExecutorOptions{
|
||||
Output: testutils.NewMockOutputWriter(),
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ type TemplateInfo struct {
|
|||
|
||||
// NewMockExecuterOptions creates a new mock executeroptions struct
|
||||
func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecutorOptions {
|
||||
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
|
||||
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
|
||||
executerOpts := &protocols.ExecutorOptions{
|
||||
TemplateID: info.ID,
|
||||
TemplateInfo: info.Info,
|
||||
|
|
@ -105,6 +105,7 @@ func (n *NoopWriter) Write(data []byte, level levels.Level) {}
|
|||
type MockOutputWriter struct {
|
||||
aurora aurora.Aurora
|
||||
RequestCallback func(templateID, url, requestType string, err error)
|
||||
FailureCallback func(result *output.InternalEvent)
|
||||
WriteCallback func(o *output.ResultEvent)
|
||||
}
|
||||
|
||||
|
|
@ -138,6 +139,9 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro
|
|||
|
||||
// WriteFailure writes the event to file and/or screen.
|
||||
func (m *MockOutputWriter) WriteFailure(result output.InternalEvent) error {
|
||||
if m.FailureCallback != nil && result != nil {
|
||||
m.FailureCallback(&result)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (m *MockOutputWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ var executerOpts protocols.ExecutorOptions
|
|||
func setup() {
|
||||
options := testutils.DefaultOptions
|
||||
testutils.Init(options)
|
||||
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
|
||||
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
|
||||
|
||||
executerOpts = protocols.ExecutorOptions{
|
||||
Output: testutils.NewMockOutputWriter(),
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ var executerOpts protocols.ExecutorOptions
|
|||
func setup() {
|
||||
options := testutils.DefaultOptions
|
||||
testutils.Init(options)
|
||||
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
|
||||
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
|
||||
|
||||
executerOpts = protocols.ExecutorOptions{
|
||||
Output: testutils.NewMockOutputWriter(),
|
||||
|
|
|
|||
|
|
@ -215,7 +215,7 @@ type Options struct {
|
|||
SystemResolvers bool
|
||||
// ShowActions displays a list of all headless actions
|
||||
ShowActions bool
|
||||
// Metrics enables display of metrics via an http endpoint
|
||||
// Deprecated: Enabled by default through clistats . Metrics enables display of metrics via an http endpoint
|
||||
Metrics bool
|
||||
// Debug mode allows debugging request/responses for the engine
|
||||
Debug bool
|
||||
|
|
|
|||
Loading…
Reference in New Issue