SDK: abstracted and minimal nuclei v3 sdk (#4104)

* new sdk progress

* nuclei v3 new sdk/library

* fix TestActionGetResource broken link

* fix clistats + clustering and more

* fix lint error

* fix missing ticker

* update advanced library usage example

* fix integration tests

* misc update

* add utm_source and fix lint error

---------

Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>

.github/workflows/build-test.yml

@@ -56,6 +56,10 @@ jobs:
         run: go run -race . -l ../functional-test/targets.txt -id tech-detect,tls-version
         working-directory: v2/cmd/nuclei/
 
-      - name: Example Code Tests
+      - name: Example SDK Simple
-        run: go build .
+        run: go run .
-        working-directory: v2/examples/
+        working-directory: v2/examples/simple/
+
+      - name: Example SDK Advanced
+        run: go run .
+        working-directory: v2/examples/advanced/

README.md

@@ -379,7 +379,7 @@ We have [a discussion thread around this](https://github.com/projectdiscovery/nu
 
 ### Using Nuclei From Go Code
 
-Examples of using Nuclei From Go Code to run templates on targets are provided in the [examples](v2/examples/) folder.
+Complete guide of using Nuclei as Library/SDK is available at [lib](v2/lib/README.md)
 
 
 ### Resources

v2/cmd/integration-test/template-path.go

@@ -16,7 +16,7 @@ var templatesPathTestCases = []TestCaseInfo{
 	//template folder path issue
 	{Path: "protocols/http/get.yaml", TestCase: &folderPathTemplateTest{}},
 	//cwd
-	{Path: "./protocols/dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}},
+	{Path: "./dns/detect-dangling-cname.yaml", TestCase: &cwdTemplateTest{}},
 	//relative path
 	{Path: "dns/dns-saas-service-detection.yaml", TestCase: &relativePathTemplateTest{}},
 	//absolute path

v2/cmd/nuclei/main.go

@@ -361,7 +361,6 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"),
 		flagSet.BoolVarP(&options.StatsJSON, "stats-json", "sj", false, "display statistics in JSONL(ines) format"),
 		flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"),
-		flagSet.BoolVarP(&options.Metrics, "metrics", "m", false, "expose nuclei metrics on a port"),
 		flagSet.IntVarP(&options.MetricsPort, "metrics-port", "mp", 9092, "port to expose nuclei metrics on"),
 	)
 

v2/examples/advanced/advanced.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	nuclei "github.com/projectdiscovery/nuclei/v2/lib"
+	"github.com/remeh/sizedwaitgroup"
+)
+
+func main() {
+	// create nuclei engine with options
+	ne, err := nuclei.NewThreadSafeNucleiEngine()
+	if err != nil {
+		panic(err)
+	}
+	// setup sizedWaitgroup to handle concurrency
+	sg := sizedwaitgroup.New(10)
+
+	// scan 1 = run dns templates on scanme.sh
+	sg.Add()
+	go func() {
+		defer sg.Done()
+		err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"},
+			nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}),
+		)
+		if err != nil {
+			panic(err)
+		}
+	}()
+
+	// scan 2 = run templates with oast tags on honey.scanme.sh
+	sg.Add()
+	go func() {
+		defer sg.Done()
+		err = ne.ExecuteNucleiWithOpts([]string{"http://honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"oast"}}))
+		if err != nil {
+			panic(err)
+		}
+	}()
+
+	// wait for all scans to finish
+	sg.Wait()
+	defer ne.Close()
+
+	// Output:
+	// [dns-saas-service-detection] scanme.sh
+	// [nameserver-fingerprint] scanme.sh
+	// [dns-saas-service-detection] honey.scanme.sh
+}

v2/examples/simple.go

@@ -1,106 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/logrusorgru/aurora"
-
-	"github.com/projectdiscovery/goflags"
-	"github.com/projectdiscovery/httpx/common/httpx"
-	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
-	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
-	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
-	"github.com/projectdiscovery/nuclei/v2/pkg/core"
-	"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs"
-	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
-	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
-	"github.com/projectdiscovery/nuclei/v2/pkg/types"
-	"github.com/projectdiscovery/ratelimit"
-)
-
-func main() {
-	cache := hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount, nil)
-	defer cache.Close()
-
-	mockProgress := &testutils.MockProgressClient{}
-	reportingClient, _ := reporting.New(&reporting.Options{}, "")
-	defer reportingClient.Close()
-
-	outputWriter := testutils.NewMockOutputWriter()
-	outputWriter.WriteCallback = func(event *output.ResultEvent) {
-		fmt.Printf("Got Result: %v\n", event)
-	}
-
-	defaultOpts := types.DefaultOptions()
-	protocolstate.Init(defaultOpts)
-	protocolinit.Init(defaultOpts)
-
-	defaultOpts.IncludeIds = goflags.StringSlice{"cname-service", "tech-detect"}
-	defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags
-
-	interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress)
-	interactClient, err := interactsh.New(interactOpts)
-	if err != nil {
-		log.Fatalf("Could not create interact client: %s\n", err)
-	}
-	defer interactClient.Close()
-
-	home, _ := os.UserHomeDir()
-	catalog := disk.NewCatalog(filepath.Join(home, "nuclei-templates"))
-	executerOpts := protocols.ExecutorOptions{
-		Output:          outputWriter,
-		Options:         defaultOpts,
-		Progress:        mockProgress,
-		Catalog:         catalog,
-		IssuesClient:    reportingClient,
-		RateLimiter:     ratelimit.New(context.Background(), 150, time.Second),
-		Interactsh:      interactClient,
-		HostErrorsCache: cache,
-		Colorizer:       aurora.NewAurora(true),
-		ResumeCfg:       types.NewResumeCfg(),
-	}
-	engine := core.New(defaultOpts)
-	engine.SetExecuterOptions(executerOpts)
-
-	workflowLoader, err := parsers.NewLoader(&executerOpts)
-	if err != nil {
-		log.Fatalf("Could not create workflow loader: %s\n", err)
-	}
-	executerOpts.WorkflowLoader = workflowLoader
-
-	store, err := loader.New(loader.NewConfig(defaultOpts, catalog, executerOpts))
-	if err != nil {
-		log.Fatalf("Could not create loader client: %s\n", err)
-	}
-	store.Load()
-
-	// flat input without probe
-	inputArgs := []*contextargs.MetaInput{{Input: "docs.hackerone.com"}}
-	input := &inputs.SimpleInputProvider{Inputs: inputArgs}
-
-	httpxOptions := httpx.DefaultOptions
-	httpxOptions.Timeout = 5 * time.Second
-	httpxClient, err := httpx.New(&httpxOptions)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	// use httpx to probe the URL => https://scanme.sh
-	input.SetWithProbe("scanme.sh", httpxClient)
-
-	_ = engine.Execute(store.Templates(), input)
-	engine.WorkPool().Wait() // Wait for the scan to finish
-}

v2/examples/simple/simple.go

@@ -0,0 +1,20 @@
+package main
+
+import nuclei "github.com/projectdiscovery/nuclei/v2/lib"
+
+func main() {
+	ne, err := nuclei.NewNucleiEngine(
+		nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"oast"}}),
+		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}), // optionally enable metrics server for better observability
+	)
+	if err != nil {
+		panic(err)
+	}
+	// load targets and optionally probe non http/https targets
+	ne.LoadTargets([]string{"http://honey.scanme.sh"}, false)
+	err = ne.ExecuteWithCallback(nil)
+	if err != nil {
+		panic(err)
+	}
+	defer ne.Close()
+}

v2/internal/installer/util.go

@@ -103,3 +103,29 @@ func isEmptyDir(dir string) bool {
 	})
 	return !hasFiles
 }
+
+// getUtmSource returns utm_source from environment variable
+func getUtmSource() string {
+	value := ""
+	switch {
+	case os.Getenv("GH_ACTION") != "":
+		value = "ghci"
+	case os.Getenv("TRAVIS") != "":
+		value = "travis"
+	case os.Getenv("CIRCLECI") != "":
+		value = "circleci"
+	case os.Getenv("CI") != "":
+		value = "gitlabci" // this also includes bitbucket
+	case os.Getenv("GITHUB_ACTIONS") != "":
+		value = "ghci"
+	case os.Getenv("AWS_EXECUTION_ENV") != "":
+		value = os.Getenv("AWS_EXECUTION_ENV")
+	case os.Getenv("JENKINS_URL") != "":
+		value = "jenkins"
+	case os.Getenv("FUNCTION_TARGET") != "":
+		value = "gcf"
+	default:
+		value = "unknown"
+	}
+	return value
+}

v2/internal/installer/versioncheck.go

@@ -6,6 +6,7 @@ import (
 	"net/url"
 	"os"
 	"runtime"
+	"sync"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/retryablehttp-go"
@@ -34,7 +35,56 @@ type PdtmAPIResponse struct {
 // and returns an error if it fails to check on success it returns nil and changes are
 // made to the default config in config.DefaultConfig
 func NucleiVersionCheck() error {
-	resp, err := retryableHttpClient.Get(pdtmNucleiVersionEndpoint + "?" + getpdtmParams())
+	return doVersionCheck(false)
+}
+
+// this will be updated by features of 1.21 release (which directly provides sync.Once(func()))
+type sdkUpdateCheck struct {
+	sync.Once
+}
+
+var sdkUpdateCheckInstance = &sdkUpdateCheck{}
+
+// NucleiSDKVersionCheck checks for latest version of nuclei which running in sdk mode
+// this only happens once per process regardless of how many times this function is called
+func NucleiSDKVersionCheck() {
+	sdkUpdateCheckInstance.Do(func() {
+		_ = doVersionCheck(true)
+	})
+}
+
+// getpdtmParams returns encoded query parameters sent to update check endpoint
+func getpdtmParams(isSDK bool) string {
+	params := &url.Values{}
+	params.Add("os", runtime.GOOS)
+	params.Add("arch", runtime.GOARCH)
+	params.Add("go_version", runtime.Version())
+	params.Add("v", config.Version)
+	if isSDK {
+		params.Add("sdk", "true")
+	}
+	params.Add("utm_source", getUtmSource())
+	return params.Encode()
+}
+
+// UpdateIgnoreFile updates default ignore file by downloading latest ignore file
+func UpdateIgnoreFile() error {
+	resp, err := retryableHttpClient.Get(pdtmNucleiIgnoreFileEndpoint + "?" + getpdtmParams(false))
+	if err != nil {
+		return err
+	}
+	bin, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(config.DefaultConfig.GetIgnoreFilePath(), bin, 0644); err != nil {
+		return err
+	}
+	return config.DefaultConfig.UpdateNucleiIgnoreHash()
+}
+
+func doVersionCheck(isSDK bool) error {
+	resp, err := retryableHttpClient.Get(pdtmNucleiVersionEndpoint + "?" + getpdtmParams(isSDK))
 	if err != nil {
 		return err
 	}
@@ -63,29 +113,3 @@ func NucleiVersionCheck() error {
 	}
 	return config.DefaultConfig.WriteVersionCheckData(pdtmResp.IgnoreHash, nucleiversion, templateversion)
 }
-
-// getpdtmParams returns encoded query parameters sent to update check endpoint
-func getpdtmParams() string {
-	params := &url.Values{}
-	params.Add("os", runtime.GOOS)
-	params.Add("arch", runtime.GOARCH)
-	params.Add("go_version", runtime.Version())
-	params.Add("v", config.Version)
-	return params.Encode()
-}
-
-// UpdateIgnoreFile updates default ignore file by downloading latest ignore file
-func UpdateIgnoreFile() error {
-	resp, err := retryableHttpClient.Get(pdtmNucleiIgnoreFileEndpoint + "?" + getpdtmParams())
-	if err != nil {
-		return err
-	}
-	bin, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	if err := os.WriteFile(config.DefaultConfig.GetIgnoreFilePath(), bin, 0644); err != nil {
-		return err
-	}
-	return config.DefaultConfig.UpdateNucleiIgnoreHash()
-}

v2/internal/runner/options.go

@@ -71,7 +71,7 @@ func ParseOptions(options *types.Options) {
 	}
 	// Validate the options passed by the user and if any
 	// invalid options have been used, exit.
-	if err := validateOptions(options); err != nil {
+	if err := ValidateOptions(options); err != nil {
 		gologger.Fatal().Msgf("Program exiting: %s\n", err)
 	}
 
@@ -105,7 +105,7 @@ func ParseOptions(options *types.Options) {
 }
 
 // validateOptions validates the configuration options passed
-func validateOptions(options *types.Options) error {
+func ValidateOptions(options *types.Options) error {
 	validate := validator.New()
 	if err := validate.Struct(options); err != nil {
 		if _, ok := err.(*validator.InvalidValidationError); ok {

v2/internal/runner/runner.go

@@ -260,7 +260,7 @@ func New(options *types.Options) (*Runner, error) {
 		statsInterval = -1
 		options.EnableProgressBar = true
 	}
-	runner.progress, progressErr = progress.NewStatsTicker(statsInterval, options.EnableProgressBar, options.StatsJSON, options.Metrics, options.Cloud, options.MetricsPort)
+	runner.progress, progressErr = progress.NewStatsTicker(statsInterval, options.EnableProgressBar, options.StatsJSON, options.Cloud, options.MetricsPort)
 	if progressErr != nil {
 		return nil, progressErr
 	}
@@ -662,16 +662,6 @@ func (r *Runner) executeSmartWorkflowInput(executorOpts protocols.ExecutorOption
 }
 
 func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine) (*atomic.Bool, error) {
-	var unclusteredRequests int64
-	for _, template := range store.Templates() {
-		// workflows will dynamically adjust the totals while running, as
-		// it can't be known in advance which requests will be called
-		if len(template.Workflows) > 0 {
-			continue
-		}
-		unclusteredRequests += int64(template.TotalRequests) * r.hmapInputProvider.Count()
-	}
-
 	if r.options.VerboseVerbose {
 		for _, template := range store.Templates() {
 			r.logAvailableTemplate(template.Path)
@@ -681,34 +671,15 @@ func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine)
 		}
 	}
 
-	// Cluster the templates first because we want info on how many
+	finalTemplates := []*templates.Template{}
-	// templates did we cluster for showing to user in CLI
+	finalTemplates = append(finalTemplates, store.Templates()...)
-	originalTemplatesCount := len(store.Templates())
-	finalTemplates, clusterCount := templates.ClusterTemplates(store.Templates(), engine.ExecuterOptions())
 	finalTemplates = append(finalTemplates, store.Workflows()...)
 
-	var totalRequests int64
+	if len(finalTemplates) == 0 {
-	for _, t := range finalTemplates {
+		return nil, errors.New("no templates provided for scan")
-		if len(t.Workflows) > 0 {
-			continue
-		}
-		totalRequests += int64(t.Executer.Requests()) * r.hmapInputProvider.Count()
-	}
-	if totalRequests < unclusteredRequests {
-		gologger.Info().Msgf("Templates clustered: %d (Reduced %d Requests)", clusterCount, unclusteredRequests-totalRequests)
-	}
-	workflowCount := len(store.Workflows())
-	templateCount := originalTemplatesCount + workflowCount
-
-	// 0 matches means no templates were found in the directory
-	if templateCount == 0 {
-		return &atomic.Bool{}, errors.New("no valid templates were found")
 	}
 
-	// tracks global progress and captures stdout/stderr until p.Wait finishes
+	results := engine.ExecuteScanWithOpts(finalTemplates, r.hmapInputProvider, r.options.DisableClustering)
-	r.progress.Init(r.hmapInputProvider.Count(), templateCount, totalRequests)
-
-	results := engine.ExecuteScanWithOpts(finalTemplates, r.hmapInputProvider, true)
 	return results, nil
 }
 

v2/lib/README.md

@@ -0,0 +1,87 @@
+## Using Nuclei as Library
+
+Nuclei was primarily built as a CLI tool, but with increasing choice of users wanting to use nuclei as library in their own automation, we have added a simplified Library/SDK of nuclei in v3
+
+### Installation
+
+To add nuclei as a library to your go project, you can use the following command:
+
+```bash
+go get -u github.com/projectdiscovery/nuclei/v2/lib
+```
+
+Or add below import to your go file and let IDE handle the rest:
+
+```go
+import nuclei "github.com/projectdiscovery/nuclei/v2/lib"
+```
+
+## Basic Example of using Nuclei Library/SDK
+
+```go
+// create nuclei engine with options
+	ne, err := nuclei.NewNucleiEngine(
+		nuclei.WithTemplateFilters(nuclei.TemplateFilters{Severity: "critical"}), // run critical severity templates only
+	)
+	if err != nil {
+		panic(err)
+	}
+	// load targets and optionally probe non http/https targets
+	ne.LoadTargets([]string{"scanme.sh"}, false)
+	err = ne.ExecuteWithCallback(nil)
+	if err != nil {
+		panic(err)
+	}
+	defer ne.Close()
+```
+
+## Advanced Example of using Nuclei Library/SDK
+
+For Various use cases like batching etc you might want to run nuclei in goroutines this can be done by using `nuclei.NewThreadSafeNucleiEngine`
+
+```go
+// create nuclei engine with options
+	ne, err := nuclei.NewThreadSafeNucleiEngine()
+	if err != nil{
+        panic(err)
+    }
+	// setup waitgroup to handle concurrency
+	wg := &sync.WaitGroup{}
+
+	// scan 1 = run dns templates on scanme.sh
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "http"}))
+		if err != nil {
+            panic(err)
+        }
+	}()
+
+	// scan 2 = run http templates on honey.scanme.sh
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
+		if err != nil {
+            panic(err)
+        }
+	}()
+
+	// wait for all scans to finish
+	wg.Wait()
+	defer ne.Close()
+```
+
+## More Documentation
+
+For complete documentation of nuclei library, please refer to [godoc](https://pkg.go.dev/github.com/projectdiscovery/nuclei/v2/lib) which contains all available options and methods.
+
+
+
+### Note
+
+| :exclamation:  **Disclaimer**  |
+|---------------------------------|
+| **This project is in active development**. Expect breaking changes with releases. Review the release changelog before updating. |
+| This project was primarily built to be used as a standalone CLI tool. **Running nuclei as a service may pose security risks.** It's recommended to use with caution and additional security measures. |

v2/lib/config.go

@@ -0,0 +1,298 @@
+package nuclei
+
+import (
+	"context"
+	"time"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
+	"github.com/projectdiscovery/ratelimit"
+)
+
+// config contains all SDK configuration options
+type TemplateFilters struct {
+	Severity             string   // filter by severities (accepts CSV values of info, low, medium, high, critical)
+	ExcludeSeverities    string   // filter by excluding severities (accepts CSV values of info, low, medium, high, critical)
+	ProtocolTypes        string   // filter by protocol types
+	ExcludeProtocolTypes string   // filter by excluding protocol types
+	Authors              []string // fiter by author
+	Tags                 []string // filter by tags present in template
+	ExcludeTags          []string // filter by excluding tags present in template
+	IncludeTags          []string // filter by including tags present in template
+	IDs                  []string // filter by template IDs
+	ExcludeIDs           []string // filter by excluding template IDs
+	TemplateCondition    []string // DSL condition/ expression
+}
+
+// WithTemplateFilters sets template filters and only templates matching the filters will be
+// loaded and executed
+func WithTemplateFilters(filters TemplateFilters) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		s := severity.Severities{}
+		if err := s.Set(filters.Severity); err != nil {
+			return err
+		}
+		es := severity.Severities{}
+		if err := es.Set(filters.ExcludeSeverities); err != nil {
+			return err
+		}
+		pt := types.ProtocolTypes{}
+		if err := pt.Set(filters.ProtocolTypes); err != nil {
+			return err
+		}
+		ept := types.ProtocolTypes{}
+		if err := ept.Set(filters.ExcludeProtocolTypes); err != nil {
+			return err
+		}
+		e.opts.Authors = filters.Authors
+		e.opts.Tags = filters.Tags
+		e.opts.ExcludeTags = filters.ExcludeTags
+		e.opts.IncludeTags = filters.IncludeTags
+		e.opts.IncludeIds = filters.IDs
+		e.opts.ExcludeIds = filters.ExcludeIDs
+		e.opts.Severities = s
+		e.opts.ExcludeSeverities = es
+		e.opts.Protocols = pt
+		e.opts.ExcludeProtocols = ept
+		e.opts.IncludeConditions = filters.TemplateCondition
+		return nil
+	}
+}
+
+// InteractshOpts contains options for interactsh
+type InteractshOpts interactsh.Options
+
+// WithInteractshOptions sets interactsh options
+func WithInteractshOptions(opts InteractshOpts) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		if e.mode == threadSafe {
+			return ErrOptionsNotSupported.Msgf("WithInteractshOptions")
+		}
+		optsPtr := &opts
+		e.interactshOpts = (*interactsh.Options)(optsPtr)
+		return nil
+	}
+}
+
+// Concurrency options
+type Concurrency struct {
+	TemplateConcurrency         int // number of templates to run concurrently (per host in host-spray mode)
+	HostConcurrency             int // number of hosts to scan concurrently  (per template in template-spray mode)
+	HeadlessHostConcurrency     int // number of hosts to scan concurrently for headless templates  (per template in template-spray mode)
+	HeadlessTemplateConcurrency int // number of templates to run concurrently for headless templates (per host in host-spray mode)
+}
+
+// WithConcurrency sets concurrency options
+func WithConcurrency(opts Concurrency) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		e.opts.TemplateThreads = opts.TemplateConcurrency
+		e.opts.BulkSize = opts.HostConcurrency
+		e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency
+		e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency
+		return nil
+	}
+}
+
+// WithGlobalRateLimit sets global rate (i.e all hosts combined) limit options
+func WithGlobalRateLimit(maxTokens int, duration time.Duration) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		e.rateLimiter = ratelimit.New(context.Background(), uint(maxTokens), duration)
+		return nil
+	}
+}
+
+// HeadlessOpts contains options for headless templates
+type HeadlessOpts struct {
+	PageTimeout     int // timeout for page load
+	ShowBrowser     bool
+	HeadlessOptions []string
+	UseChrome       bool
+}
+
+// EnableHeadless allows execution of headless templates
+// *Use With Caution*: Enabling headless mode may open up attack surface due to browser usage
+// and can be prone to exploitation by custom unverified templates if not properly configured
+func EnableHeadlessWithOpts(hopts *HeadlessOpts) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		e.opts.Headless = true
+		if hopts != nil {
+			e.opts.HeadlessOptionalArguments = hopts.HeadlessOptions
+			e.opts.PageTimeout = hopts.PageTimeout
+			e.opts.ShowBrowser = hopts.ShowBrowser
+			e.opts.UseInstalledChrome = hopts.UseChrome
+		}
+		if engine.MustDisableSandbox() {
+			gologger.Warning().Msgf("The current platform and privileged user will run the browser without sandbox\n")
+		}
+		browser, err := engine.New(e.opts)
+		if err != nil {
+			return err
+		}
+		e.executerOpts.Browser = browser
+		return nil
+	}
+}
+
+// StatsOptions
+type StatsOptions struct {
+	Interval         int
+	JSON             bool
+	MetricServerPort int
+}
+
+// EnableStats enables Stats collection with defined interval(in sec) and callback
+// Note: callback is executed in a separate goroutine
+func EnableStatsWithOpts(opts StatsOptions) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		if e.mode == threadSafe {
+			return ErrOptionsNotSupported.Msgf("EnableStatsWithOpts")
+		}
+		if opts.Interval == 0 {
+			opts.Interval = 5 //sec
+		}
+		e.opts.StatsInterval = opts.Interval
+		e.enableStats = true
+		e.opts.StatsJSON = opts.JSON
+		e.opts.MetricsPort = opts.MetricServerPort
+		return nil
+	}
+}
+
+// VerbosityOptions
+type VerbosityOptions struct {
+	Verbose       bool // show verbose output
+	Silent        bool // show only results
+	Debug         bool // show debug output
+	DebugRequest  bool // show request in debug output
+	DebugResponse bool // show response in debug output
+	ShowVarDump   bool // show variable dumps in output
+}
+
+// WithVerbosity allows setting verbosity options of (internal) nuclei engine
+// and does not affect SDK output
+func WithVerbosity(opts VerbosityOptions) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		if e.mode == threadSafe {
+			return ErrOptionsNotSupported.Msgf("WithVerbosity")
+		}
+		e.opts.Verbose = opts.Verbose
+		e.opts.Silent = opts.Silent
+		e.opts.Debug = opts.Debug
+		e.opts.DebugRequests = opts.DebugRequest
+		e.opts.DebugResponse = opts.DebugResponse
+		if opts.ShowVarDump {
+			vardump.EnableVarDump = true
+		}
+		return nil
+	}
+}
+
+// NetworkConfig contains network config options
+// ex: retries , httpx probe , timeout etc
+type NetworkConfig struct {
+	Timeout           int      // Timeout in seconds
+	Retries           int      // Number of retries
+	LeaveDefaultPorts bool     // Leave default ports for http/https
+	MaxHostError      int      // Maximum number of host errors to allow before skipping that host
+	TrackError        []string // Adds given errors to max host error watchlist
+	DisableMaxHostErr bool     // Disable max host error optimization (Hosts are not skipped even if they are not responding)
+}
+
+// WithNetworkConfig allows setting network config options
+func WithNetworkConfig(opts NetworkConfig) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		if e.mode == threadSafe {
+			return ErrOptionsNotSupported.Msgf("WithNetworkConfig")
+		}
+		e.opts.Timeout = opts.Timeout
+		e.opts.Retries = opts.Retries
+		e.opts.LeaveDefaultPorts = opts.LeaveDefaultPorts
+		e.hostErrCache = hosterrorscache.New(opts.MaxHostError, hosterrorscache.DefaultMaxHostsCount, opts.TrackError)
+		return nil
+	}
+}
+
+// WithProxy allows setting proxy options
+func WithProxy(proxy []string, proxyInternalRequests bool) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		if e.mode == threadSafe {
+			return ErrOptionsNotSupported.Msgf("WithProxy")
+		}
+		e.opts.Proxy = proxy
+		e.opts.ProxyInternal = proxyInternalRequests
+		return nil
+	}
+}
+
+// WithScanStrategy allows setting scan strategy options
+func WithScanStrategy(strategy string) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		e.opts.ScanStrategy = strategy
+		return nil
+	}
+}
+
+// OutputWriter
+type OutputWriter output.Writer
+
+// UseWriter allows setting custom output writer
+// by default a mock writer is used with user defined callback
+// if outputWriter is used callback will be ignored
+func UseOutputWriter(writer OutputWriter) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		if e.mode == threadSafe {
+			return ErrOptionsNotSupported.Msgf("UseOutputWriter")
+		}
+		e.customWriter = writer
+		return nil
+	}
+}
+
+// StatsWriter
+type StatsWriter progress.Progress
+
+// UseStatsWriter allows setting a custom stats writer
+// which can be used to write stats somewhere (ex: send to webserver etc)
+func UseStatsWriter(writer StatsWriter) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		if e.mode == threadSafe {
+			return ErrOptionsNotSupported.Msgf("UseStatsWriter")
+		}
+		e.customProgress = writer
+		return nil
+	}
+}
+
+// WithTemplateUpdateCallback allows setting a callback which will be called
+// when nuclei templates are outdated
+// Note: Nuclei-templates are crucial part of nuclei and using outdated templates or nuclei sdk is not recommended
+// as it may cause unexpected results due to compatibility issues
+func WithTemplateUpdateCallback(disableTemplatesAutoUpgrade bool, callback func(newVersion string)) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		if e.mode == threadSafe {
+			return ErrOptionsNotSupported.Msgf("WithTemplateUpdateCallback")
+		}
+		e.disableTemplatesAutoUpgrade = disableTemplatesAutoUpgrade
+		e.onUpdateAvailableCallback = callback
+		return nil
+	}
+}
+
+// WithSandboxOptions allows setting supported sandbox options
+func WithSandboxOptions(allowLocalFileAccess bool, restrictLocalNetworkAccess bool) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		if e.mode == threadSafe {
+			return ErrOptionsNotSupported.Msgf("WithSandboxOptions")
+		}
+		e.opts.AllowLocalFileAccess = allowLocalFileAccess
+		e.opts.RestrictLocalNetworkAccess = restrictLocalNetworkAccess
+		return nil
+	}
+}

v2/lib/example_test.go

@@ -0,0 +1,85 @@
+//go:build !race
+// +build !race
+
+package nuclei_test
+
+import (
+	"os"
+	"testing"
+
+	nuclei "github.com/projectdiscovery/nuclei/v2/lib"
+	"github.com/remeh/sizedwaitgroup"
+)
+
+// A very simple example on how to use nuclei engine
+func ExampleNucleiEngine() {
+	// create nuclei engine with options
+	ne, err := nuclei.NewNucleiEngine(
+		nuclei.WithTemplateFilters(nuclei.TemplateFilters{IDs: []string{"self-signed-ssl"}}), // only run self-signed-ssl template
+	)
+	if err != nil {
+		panic(err)
+	}
+	// load targets and optionally probe non http/https targets
+	ne.LoadTargets([]string{"scanme.sh"}, false)
+	// when callback is nil it nuclei will print JSON output to stdout
+	err = ne.ExecuteWithCallback(nil)
+	if err != nil {
+		panic(err)
+	}
+	defer ne.Close()
+
+	// Output:
+	// [self-signed-ssl] scanme.sh:443
+}
+
+func ExampleThreadSafeNucleiEngine() {
+	// create nuclei engine with options
+	ne, err := nuclei.NewThreadSafeNucleiEngine()
+	if err != nil {
+		panic(err)
+	}
+	// setup sizedWaitgroup to handle concurrency
+	// here we are using sizedWaitgroup to limit concurrency to 1
+	sg := sizedwaitgroup.New(1)
+
+	// scan 1 = run dns templates on scanme.sh
+	sg.Add()
+	go func() {
+		defer sg.Done()
+		err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"},
+			nuclei.WithTemplateFilters(nuclei.TemplateFilters{IDs: []string{"nameserver-fingerprint"}}), // only run self-signed-ssl template
+		)
+		if err != nil {
+			panic(err)
+		}
+	}()
+
+	// scan 2 = run dns templates on honey.scanme.sh
+	sg.Add()
+	go func() {
+		defer sg.Done()
+		err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
+		if err != nil {
+			panic(err)
+		}
+	}()
+
+	// wait for all scans to finish
+	sg.Wait()
+	defer ne.Close()
+
+	// Output:
+	// [nameserver-fingerprint] scanme.sh
+	// [dns-saas-service-detection] honey.scanme.sh
+}
+
+func TestMain(m *testing.M) {
+	// this file only contains testtables examples https://go.dev/blog/examples
+	// and actual functionality test are in sdk_test.go
+	if os.Getenv("GH_ACTION") != "" {
+		// no need to run this test on github actions
+		return
+	}
+	m.Run()
+}

v2/lib/helper.go

@@ -0,0 +1,33 @@
+package nuclei
+
+import (
+	"context"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+	uncoverNuclei "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/uncover"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/uncover"
+)
+
+// helper.go file proxy execution of all nuclei functions that are nested deep inside multiple packages
+// but are helpful / useful while using nuclei as a library
+
+// GetTargetsFromUncover returns targets from uncover in given format .
+// supported formats are any string with [ip,host,port,url] placeholders
+func GetTargetsFromUncover(ctx context.Context, outputFormat string, opts *uncover.Options) (chan string, error) {
+	return uncoverNuclei.GetTargetsFromUncover(ctx, outputFormat, opts)
+}
+
+// GetTargetsFromTemplateMetadata returns all targets by querying engine metadata (ex: fofo-query,shodan-query) etc from given templates .
+// supported formats are any string with [ip,host,port,url] placeholders
+func GetTargetsFromTemplateMetadata(ctx context.Context, templates []*templates.Template, outputFormat string, opts *uncover.Options) chan string {
+	return uncoverNuclei.GetUncoverTargetsFromMetadata(ctx, templates, outputFormat, opts)
+}
+
+// DefaultConfig is instance of default nuclei configs
+// any mutations to this config will be reflected in all nuclei instances (saves some config to disk)
+var DefaultConfig *config.Config
+
+func init() {
+	DefaultConfig = config.DefaultConfig
+}

v2/lib/multi.go

@@ -0,0 +1,152 @@
+package nuclei
+
+import (
+	"context"
+	"time"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
+	"github.com/projectdiscovery/nuclei/v2/pkg/core"
+	"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"github.com/projectdiscovery/ratelimit"
+	errorutil "github.com/projectdiscovery/utils/errors"
+)
+
+// unsafeOptions are those nuclei objects/instances/types
+// that are required to run nuclei engine but are not thread safe
+// hence they are ephemeral and are created on every ExecuteNucleiWithOpts invocation
+// in ThreadSafeNucleiEngine
+type unsafeOptions struct {
+	executerOpts protocols.ExecutorOptions
+	engine       *core.Engine
+}
+
+// createEphemeralObjects creates ephemeral nuclei objects/instances/types
+func createEphemeralObjects(base *NucleiEngine, opts *types.Options) (*unsafeOptions, error) {
+	u := &unsafeOptions{}
+	u.executerOpts = protocols.ExecutorOptions{
+		Output:          base.customWriter,
+		Options:         opts,
+		Progress:        base.customProgress,
+		Catalog:         base.catalog,
+		IssuesClient:    base.rc,
+		RateLimiter:     base.rateLimiter,
+		Interactsh:      base.interactshClient,
+		HostErrorsCache: base.hostErrCache,
+		Colorizer:       aurora.NewAurora(true),
+		ResumeCfg:       types.NewResumeCfg(),
+	}
+	if opts.RateLimitMinute > 0 {
+		u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimitMinute), time.Minute)
+	} else if opts.RateLimit > 0 {
+		u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimit), time.Second)
+	} else {
+		u.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
+	}
+	u.engine = core.New(opts)
+	u.engine.SetExecuterOptions(u.executerOpts)
+	return u, nil
+}
+
+// ThreadSafeNucleiEngine is a tweaked version of nuclei.Engine whose methods are thread-safe
+// and can be used concurrently. Non-thread-safe methods start with Global prefix
+type ThreadSafeNucleiEngine struct {
+	eng *NucleiEngine
+}
+
+// NewThreadSafeNucleiEngine creates a new nuclei engine with given options
+// whose methods are thread-safe and can be used concurrently
+// Note: Non-thread-safe methods start with Global prefix
+func NewThreadSafeNucleiEngine(opts ...NucleiSDKOptions) (*ThreadSafeNucleiEngine, error) {
+	// default options
+	e := &NucleiEngine{
+		opts: types.DefaultOptions(),
+		mode: threadSafe,
+	}
+	for _, option := range opts {
+		if err := option(e); err != nil {
+			return nil, err
+		}
+	}
+	if err := e.init(); err != nil {
+		return nil, err
+	}
+	return &ThreadSafeNucleiEngine{eng: e}, nil
+}
+
+// GlobalLoadAllTemplates loads all templates from nuclei-templates repo
+// This method will load all templates based on filters given at the time of nuclei engine creation in opts
+func (e *ThreadSafeNucleiEngine) GlobalLoadAllTemplates() error {
+	return e.eng.LoadAllTemplates()
+}
+
+// GlobalResultCallback sets a callback function which will be called for each result
+func (e *ThreadSafeNucleiEngine) GlobalResultCallback(callback func(event *output.ResultEvent)) {
+	e.eng.resultCallbacks = []func(*output.ResultEvent){callback}
+}
+
+// ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found)
+// This method can be called concurrently and it will use some global resources but can be runned parllely
+// by invoking this method with different options and targets
+// Note: Not all options are thread-safe. this method will throw error if you try to use non-thread-safe options
+func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ...NucleiSDKOptions) error {
+	baseOpts := *e.eng.opts
+	tmpEngine := &NucleiEngine{opts: &baseOpts, mode: threadSafe}
+	for _, option := range opts {
+		if err := option(tmpEngine); err != nil {
+			return err
+		}
+	}
+	// create ephemeral nuclei objects/instances/types using base nuclei engine
+	unsafeOpts, err := createEphemeralObjects(e.eng, tmpEngine.opts)
+	if err != nil {
+		return err
+	}
+
+	// load templates
+	workflowLoader, err := parsers.NewLoader(&unsafeOpts.executerOpts)
+	if err != nil {
+		return errorutil.New("Could not create workflow loader: %s\n", err)
+	}
+	unsafeOpts.executerOpts.WorkflowLoader = workflowLoader
+
+	store, err := loader.New(loader.NewConfig(tmpEngine.opts, e.eng.catalog, unsafeOpts.executerOpts))
+	if err != nil {
+		return errorutil.New("Could not create loader client: %s\n", err)
+	}
+	store.Load()
+
+	inputProvider := &inputs.SimpleInputProvider{
+		Inputs: []*contextargs.MetaInput{},
+	}
+
+	// load targets
+	for _, target := range targets {
+		inputProvider.Set(target)
+	}
+
+	if len(store.Templates()) == 0 && len(store.Workflows()) == 0 {
+		return ErrNoTemplatesAvailable
+	}
+	if inputProvider.Count() == 0 {
+		return ErrNoTargetsAvailable
+	}
+
+	engine := core.New(tmpEngine.opts)
+	engine.SetExecuterOptions(unsafeOpts.executerOpts)
+
+	_ = engine.ExecuteScanWithOpts(store.Templates(), inputProvider, false)
+
+	engine.WorkPool().Wait()
+	return nil
+}
+
+// Close all resources used by nuclei engine
+func (e *ThreadSafeNucleiEngine) Close() {
+	e.eng.Close()
+}

v2/lib/sdk.go

@@ -0,0 +1,180 @@
+package nuclei
+
+import (
+	"bufio"
+	"io"
+
+	"github.com/projectdiscovery/httpx/common/httpx"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
+	"github.com/projectdiscovery/nuclei/v2/pkg/core"
+	"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"github.com/projectdiscovery/ratelimit"
+	"github.com/projectdiscovery/retryablehttp-go"
+	errorutil "github.com/projectdiscovery/utils/errors"
+)
+
+// NucleiSDKOptions contains options for nuclei SDK
+type NucleiSDKOptions func(e *NucleiEngine) error
+
+var (
+	// ErrNotImplemented is returned when a feature is not implemented
+	ErrNotImplemented = errorutil.New("Not implemented")
+	// ErrNoTemplatesAvailable is returned when no templates are available to execute
+	ErrNoTemplatesAvailable = errorutil.New("No templates available")
+	// ErrNoTargetsAvailable is returned when no targets are available to scan
+	ErrNoTargetsAvailable = errorutil.New("No targets available")
+	// ErrOptionsNotSupported is returned when an option is not supported in thread safe mode
+	ErrOptionsNotSupported = errorutil.NewWithFmt("Option %v not supported in thread safe mode")
+)
+
+type engineMode uint
+
+const (
+	singleInstance engineMode = iota
+	threadSafe
+)
+
+// NucleiEngine is the Engine/Client for nuclei which
+// runs scans using templates and returns results
+type NucleiEngine struct {
+	// user options
+	resultCallbacks             []func(event *output.ResultEvent)
+	onFailureCallback           func(event *output.InternalEvent)
+	disableTemplatesAutoUpgrade bool
+	enableStats                 bool
+	onUpdateAvailableCallback   func(newVersion string)
+
+	// ready-status fields
+	templatesLoaded bool
+
+	// unexported core fields
+	interactshClient *interactsh.Client
+	catalog          *disk.DiskCatalog
+	rateLimiter      *ratelimit.Limiter
+	store            *loader.Store
+	httpxClient      *httpx.HTTPX
+	inputProvider    *inputs.SimpleInputProvider
+	engine           *core.Engine
+	mode             engineMode
+	browserInstance  *engine.Browser
+	httpClient       *retryablehttp.Client
+
+	// unexported meta options
+	opts           *types.Options
+	interactshOpts *interactsh.Options
+	hostErrCache   *hosterrorscache.Cache
+	customWriter   output.Writer
+	customProgress progress.Progress
+	rc             reporting.Client
+	executerOpts   protocols.ExecutorOptions
+}
+
+// LoadAllTemplates loads all nuclei template based on given options
+func (e *NucleiEngine) LoadAllTemplates() error {
+	workflowLoader, err := parsers.NewLoader(&e.executerOpts)
+	if err != nil {
+		return errorutil.New("Could not create workflow loader: %s\n", err)
+	}
+	e.executerOpts.WorkflowLoader = workflowLoader
+
+	e.store, err = loader.New(loader.NewConfig(e.opts, e.catalog, e.executerOpts))
+	if err != nil {
+		return errorutil.New("Could not create loader client: %s\n", err)
+	}
+	e.store.Load()
+	return nil
+}
+
+// GetTemplates returns all nuclei templates that are loaded
+func (e *NucleiEngine) GetTemplates() []*templates.Template {
+	if !e.templatesLoaded {
+		_ = e.LoadAllTemplates()
+	}
+	return e.store.Templates()
+}
+
+// LoadTargets(urls/domains/ips only) adds targets to the nuclei engine
+func (e *NucleiEngine) LoadTargets(targets []string, probeNonHttp bool) {
+	for _, target := range targets {
+		if probeNonHttp {
+			e.inputProvider.SetWithProbe(target, e.httpxClient)
+		} else {
+			e.inputProvider.Set(target)
+		}
+	}
+}
+
+// LoadTargetsFromReader adds targets(urls/domains/ips only) from reader to the nuclei engine
+func (e *NucleiEngine) LoadTargetsFromReader(reader io.Reader, probeNonHttp bool) {
+	buff := bufio.NewScanner(reader)
+	for buff.Scan() {
+		if probeNonHttp {
+			e.inputProvider.SetWithProbe(buff.Text(), e.httpxClient)
+		} else {
+			e.inputProvider.Set(buff.Text())
+		}
+	}
+}
+
+// Close all resources used by nuclei engine
+func (e *NucleiEngine) Close() {
+	e.interactshClient.Close()
+	e.rc.Close()
+	e.customWriter.Close()
+	e.hostErrCache.Close()
+	e.executerOpts.RateLimiter.Stop()
+}
+
+// ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found)
+func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.ResultEvent)) error {
+	if !e.templatesLoaded {
+		_ = e.LoadAllTemplates()
+	}
+	if len(e.store.Templates()) == 0 && len(e.store.Workflows()) == 0 {
+		return ErrNoTemplatesAvailable
+	}
+	if e.inputProvider.Count() == 0 {
+		return ErrNoTargetsAvailable
+	}
+
+	filtered := []func(event *output.ResultEvent){}
+	for _, callback := range callback {
+		if callback != nil {
+			filtered = append(filtered, callback)
+		}
+	}
+	e.resultCallbacks = append(e.resultCallbacks, filtered...)
+
+	_ = e.engine.ExecuteScanWithOpts(e.store.Templates(), e.inputProvider, false)
+	defer e.engine.WorkPool().Wait()
+	return nil
+}
+
+// NewNucleiEngine creates a new nuclei engine instance
+func NewNucleiEngine(options ...NucleiSDKOptions) (*NucleiEngine, error) {
+	// default options
+	e := &NucleiEngine{
+		opts: types.DefaultOptions(),
+		mode: singleInstance,
+	}
+	for _, option := range options {
+		if err := option(e); err != nil {
+			return nil, err
+		}
+	}
+	if err := e.init(); err != nil {
+		return nil, err
+	}
+	return e, nil
+}

v2/lib/sdk_private.go

@@ -0,0 +1,197 @@
+package nuclei
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/httpx/common/httpx"
+	"github.com/projectdiscovery/nuclei/v2/internal/installer"
+	"github.com/projectdiscovery/nuclei/v2/internal/runner"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
+	"github.com/projectdiscovery/nuclei/v2/pkg/core"
+	"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
+	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"github.com/projectdiscovery/ratelimit"
+)
+
+// applyRequiredDefaults to options
+func (e *NucleiEngine) applyRequiredDefaults() {
+	if e.customWriter == nil {
+		mockoutput := testutils.NewMockOutputWriter()
+		mockoutput.WriteCallback = func(event *output.ResultEvent) {
+			if len(e.resultCallbacks) > 0 {
+				for _, callback := range e.resultCallbacks {
+					if callback != nil {
+						callback(event)
+					}
+				}
+				return
+			}
+			sb := strings.Builder{}
+			sb.WriteString(fmt.Sprintf("[%v] ", event.TemplateID))
+			if event.Matched != "" {
+				sb.WriteString(event.Matched)
+			} else {
+				sb.WriteString(event.Host)
+			}
+			fmt.Println(sb.String())
+		}
+		if e.onFailureCallback != nil {
+			mockoutput.FailureCallback = e.onFailureCallback
+		}
+		e.customWriter = mockoutput
+	}
+	if e.customProgress == nil {
+		e.customProgress = &testutils.MockProgressClient{}
+	}
+	if e.hostErrCache == nil {
+		e.hostErrCache = hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount, nil)
+	}
+	// setup interactsh
+	if e.interactshOpts != nil {
+		e.interactshOpts.Output = e.customWriter
+		e.interactshOpts.Progress = e.customProgress
+	} else {
+		e.interactshOpts = interactsh.DefaultOptions(e.customWriter, e.rc, e.customProgress)
+	}
+	if e.rateLimiter == nil {
+		e.rateLimiter = ratelimit.New(context.Background(), 150, time.Second)
+	}
+	// these templates are known to have weak matchers
+	// and idea is to disable them to avoid false positives
+	e.opts.ExcludeTags = config.ReadIgnoreFile().Tags
+
+	e.inputProvider = &inputs.SimpleInputProvider{
+		Inputs: []*contextargs.MetaInput{},
+	}
+}
+
+// init
+func (e *NucleiEngine) init() error {
+	if e.opts.Verbose {
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
+	} else if e.opts.Debug {
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
+	} else if e.opts.Silent {
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
+	}
+
+	if err := runner.ValidateOptions(e.opts); err != nil {
+		return err
+	}
+
+	if e.opts.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" {
+		httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{})
+		if err != nil {
+			return err
+		}
+		e.httpClient = httpclient
+	}
+
+	_ = protocolstate.Init(e.opts)
+	_ = protocolinit.Init(e.opts)
+	e.applyRequiredDefaults()
+	var err error
+
+	// setup progressbar
+	if e.enableStats {
+		progressInstance, progressErr := progress.NewStatsTicker(e.opts.StatsInterval, e.enableStats, e.opts.StatsJSON, false, e.opts.MetricsPort)
+		if progressErr != nil {
+			return err
+		}
+		e.customProgress = progressInstance
+		e.interactshOpts.Progress = progressInstance
+	}
+
+	if err := reporting.CreateConfigIfNotExists(); err != nil {
+		return err
+	}
+	// we don't support reporting config in sdk mode
+	if e.rc, err = reporting.New(&reporting.Options{}, ""); err != nil {
+		return err
+	}
+	e.interactshOpts.IssuesClient = e.rc
+	if e.httpClient != nil {
+		e.interactshOpts.HTTPClient = e.httpClient
+	}
+	if e.interactshClient, err = interactsh.New(e.interactshOpts); err != nil {
+		return err
+	}
+
+	e.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory)
+
+	e.executerOpts = protocols.ExecutorOptions{
+		Output:          e.customWriter,
+		Options:         e.opts,
+		Progress:        e.customProgress,
+		Catalog:         e.catalog,
+		IssuesClient:    e.rc,
+		RateLimiter:     e.rateLimiter,
+		Interactsh:      e.interactshClient,
+		HostErrorsCache: e.hostErrCache,
+		Colorizer:       aurora.NewAurora(true),
+		ResumeCfg:       types.NewResumeCfg(),
+		Browser:         e.browserInstance,
+	}
+
+	if e.opts.RateLimitMinute > 0 {
+		e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimitMinute), time.Minute)
+	} else if e.opts.RateLimit > 0 {
+		e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), time.Second)
+	} else {
+		e.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
+	}
+
+	e.engine = core.New(e.opts)
+	e.engine.SetExecuterOptions(e.executerOpts)
+
+	httpxOptions := httpx.DefaultOptions
+	httpxOptions.Timeout = 5 * time.Second
+	if e.httpxClient, err = httpx.New(&httpxOptions); err != nil {
+		return err
+	}
+
+	// Only Happens once regardless how many times this function is called
+	// This will update ignore file to filter out templates with weak matchers to avoid false positives
+	// and also upgrade templates to latest version if available
+	installer.NucleiSDKVersionCheck()
+
+	return e.processUpdateCheckResults()
+}
+
+type syncOnce struct {
+	sync.Once
+}
+
+var updateCheckInstance = &syncOnce{}
+
+// processUpdateCheckResults processes update check results
+func (e *NucleiEngine) processUpdateCheckResults() error {
+	var err error
+	updateCheckInstance.Do(func() {
+		if e.onUpdateAvailableCallback != nil {
+			e.onUpdateAvailableCallback(config.DefaultConfig.LatestNucleiTemplatesVersion)
+		}
+		tm := installer.TemplateManager{}
+		err = tm.UpdateIfOutdated()
+	})
+	return err
+}

v2/lib/sdk_test.go

@@ -0,0 +1,42 @@
+package nuclei_test
+
+import (
+	"testing"
+
+	nuclei "github.com/projectdiscovery/nuclei/v2/lib"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSimpleNuclei(t *testing.T) {
+	ne, err := nuclei.NewNucleiEngine(
+		nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}),
+		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}),
+	)
+	require.Nil(t, err)
+	ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here
+	// when callback is nil it nuclei will print JSON output to stdout
+	err = ne.ExecuteWithCallback(nil)
+	require.Nil(t, err)
+	defer ne.Close()
+}
+
+func TestThreadSafeNuclei(t *testing.T) {
+	// create nuclei engine with options
+	ne, err := nuclei.NewThreadSafeNucleiEngine()
+	require.Nil(t, err)
+
+	// scan 1 = run dns templates on scanme.sh
+	t.Run("scanme.sh", func(t *testing.T) {
+		err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
+		require.Nil(t, err)
+	})
+
+	// scan 2 = run dns templates on honey.scanme.sh
+	t.Run("honey.scanme.sh", func(t *testing.T) {
+		err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
+		require.Nil(t, err)
+	})
+
+	// wait for all scans to finish
+	defer ne.Close()
+}

v2/pkg/core/execute_options.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/remeh/sizedwaitgroup"
 
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
@@ -34,13 +35,36 @@ func (e *Engine) ExecuteScanWithOpts(templatesList []*templates.Template, target
 	results := &atomic.Bool{}
 	selfcontainedWg := &sync.WaitGroup{}
 
+	totalReqBeforeCluster := getRequestCount(templatesList) * int(target.Count())
+
+	// attempt to cluster templates if noCluster is false
 	var finalTemplates []*templates.Template
+	clusterCount := 0
 	if !noCluster {
-		finalTemplates, _ = templates.ClusterTemplates(templatesList, e.executerOpts)
+		finalTemplates, clusterCount = templates.ClusterTemplates(templatesList, e.executerOpts)
 	} else {
 		finalTemplates = templatesList
 	}
 
+	totalReqAfterClustering := getRequestCount(finalTemplates) * int(target.Count())
+
+	if !noCluster && totalReqAfterClustering < totalReqBeforeCluster {
+		gologger.Info().Msgf("Templates clustered: %d (Reduced %d Requests)", clusterCount, totalReqBeforeCluster-totalReqAfterClustering)
+	}
+
+	// 0 matches means no templates were found in the directory
+	if len(finalTemplates) == 0 {
+		return &atomic.Bool{}
+	}
+
+	if e.executerOpts.Progress != nil {
+		// Notes:
+		// workflow requests are not counted as they can be conditional
+		// templateList count is user requested templates count (before clustering)
+		// totalReqAfterClustering is total requests count after clustering
+		e.executerOpts.Progress.Init(target.Count(), len(templatesList), int64(totalReqAfterClustering))
+	}
+
 	if stringsutil.EqualFoldAny(e.options.ScanStrategy, scanstrategy.Auto.String(), "") {
 		// TODO: this is only a placeholder, auto scan strategy should choose scan strategy
 		// based on no of hosts , templates , stream and other optimization parameters
@@ -122,3 +146,17 @@ func (e *Engine) executeHostSpray(templatesList []*templates.Template, target In
 	wp.Wait()
 	return results
 }
+
+// returns total requests count
+func getRequestCount(templates []*templates.Template) int {
+	count := 0
+	for _, template := range templates {
+		// ignore requests in workflows as total requests in workflow
+		// depends on what templates will be called in workflow
+		if len(template.Workflows) > 0 {
+			continue
+		}
+		count += template.TotalRequests
+	}
+	return count
+}

v2/pkg/core/workflow_execute_test.go

@@ -15,7 +15,7 @@ import (
 )
 
 func TestWorkflowsSimple(t *testing.T) {
-	progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
+	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
 		{Executers: []*workflows.ProtocolExecuterPair{{
@@ -29,7 +29,7 @@ func TestWorkflowsSimple(t *testing.T) {
 }
 
 func TestWorkflowsSimpleMultiple(t *testing.T) {
-	progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
+	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	var firstInput, secondInput string
 	workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
@@ -54,7 +54,7 @@ func TestWorkflowsSimpleMultiple(t *testing.T) {
 }
 
 func TestWorkflowsSubtemplates(t *testing.T) {
-	progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
+	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	var firstInput, secondInput string
 	workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
@@ -80,7 +80,7 @@ func TestWorkflowsSubtemplates(t *testing.T) {
 }
 
 func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
-	progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
+	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	var firstInput, secondInput string
 	workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
@@ -104,7 +104,7 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
 }
 
 func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
-	progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
+	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	var firstInput, secondInput string
 	workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
@@ -133,7 +133,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
 }
 
 func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
-	progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
+	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	var firstInput, secondInput string
 	workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{

v2/pkg/progress/progress.go

@@ -4,10 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"net"
-	"net/http"
 	"os"
-	"strconv"
 	"strings"
 	"time"
 
@@ -44,13 +41,12 @@ type StatsTicker struct {
 	cloud        bool
 	active       bool
 	outputJSON   bool
-	server       *http.Server
 	stats        clistats.StatisticsClient
 	tickDuration time.Duration
 }
 
 // NewStatsTicker creates and returns a new progress tracking object.
-func NewStatsTicker(duration int, active, outputJSON, metrics, cloud bool, port int) (Progress, error) {
+func NewStatsTicker(duration int, active, outputJSON, cloud bool, port int) (Progress, error) {
 	var tickDuration time.Duration
 	if active && duration != -1 {
 		tickDuration = time.Duration(duration) * time.Second
@@ -60,7 +56,12 @@ func NewStatsTicker(duration int, active, outputJSON, metrics, cloud bool, port
 
 	progress := &StatsTicker{}
 
-	stats, err := clistats.New()
+	statsOpts := &clistats.DefaultOptions
+	statsOpts.ListenPort = port
+	// metrics port is enabled by default and is not configurable with new version of clistats
+	// by default 63636 is used and than can be modified with -mp flag
+
+	stats, err := clistats.NewWithOptions(context.TODO(), statsOpts)
 	if err != nil {
 		return nil, err
 	}
@@ -70,21 +71,6 @@ func NewStatsTicker(duration int, active, outputJSON, metrics, cloud bool, port
 	progress.tickDuration = tickDuration
 	progress.outputJSON = outputJSON
 
-	if metrics {
-		http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
-			metrics := progress.getMetrics()
-			_ = json.NewEncoder(w).Encode(metrics)
-		})
-		progress.server = &http.Server{
-			Addr:    net.JoinHostPort("127.0.0.1", strconv.Itoa(port)),
-			Handler: http.DefaultServeMux,
-		}
-		go func() {
-			if err := progress.server.ListenAndServe(); err != nil {
-				gologger.Warning().Msgf("Could not serve metrics: %s", err)
-			}
-		}()
-	}
 	return progress, nil
 }
 
@@ -110,6 +96,7 @@ func (p *StatsTicker) Init(hostCount int64, rulesCount int, requestCount int64)
 			gologger.Warning().Msgf("Couldn't start statistics: %s", err)
 		}
 
+		// Note: this is needed and is responsible for the tick event
 		p.stats.GetStatResponse(p.tickDuration, func(s string, err error) error {
 			if err != nil {
 				gologger.Warning().Msgf("Could not read statistics: %s\n", err)
@@ -265,11 +252,6 @@ func metricsMap(stats clistats.StatisticsClient) map[string]interface{} {
 	return results
 }
 
-// getMetrics returns a map of important metrics for client
-func (p *StatsTicker) getMetrics() map[string]interface{} {
-	return metricsMap(p.stats)
-}
-
 // fmtDuration formats the duration for the time elapsed
 func fmtDuration(d time.Duration) string {
 	d = d.Round(time.Second)
@@ -294,7 +276,4 @@ func (p *StatsTicker) Stop() {
 			gologger.Warning().Msgf("Couldn't stop statistics: %s", err)
 		}
 	}
-	if p.server != nil {
-		_ = p.server.Shutdown(context.Background())
-	}
 }

v2/pkg/templates/compile_test.go

@@ -36,7 +36,7 @@ var executerOpts protocols.ExecutorOptions
 func setup() {
 	options := testutils.DefaultOptions
 	testutils.Init(options)
-	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
+	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	executerOpts = protocols.ExecutorOptions{
 		Output:       testutils.NewMockOutputWriter(),

v2/pkg/testutils/testutils.go

@@ -77,7 +77,7 @@ type TemplateInfo struct {
 
 // NewMockExecuterOptions creates a new mock executeroptions struct
 func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecutorOptions {
-	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
+	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
 	executerOpts := &protocols.ExecutorOptions{
 		TemplateID:   info.ID,
 		TemplateInfo: info.Info,
@@ -105,6 +105,7 @@ func (n *NoopWriter) Write(data []byte, level levels.Level) {}
 type MockOutputWriter struct {
 	aurora          aurora.Aurora
 	RequestCallback func(templateID, url, requestType string, err error)
+	FailureCallback func(result *output.InternalEvent)
 	WriteCallback   func(o *output.ResultEvent)
 }
 
@@ -138,6 +139,9 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro
 
 // WriteFailure writes the event to file and/or screen.
 func (m *MockOutputWriter) WriteFailure(result output.InternalEvent) error {
+	if m.FailureCallback != nil && result != nil {
+		m.FailureCallback(&result)
+	}
 	return nil
 }
 func (m *MockOutputWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {

v2/pkg/tmplexec/flow/flow_executor_test.go

@@ -23,7 +23,7 @@ var executerOpts protocols.ExecutorOptions
 func setup() {
 	options := testutils.DefaultOptions
 	testutils.Init(options)
-	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
+	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	executerOpts = protocols.ExecutorOptions{
 		Output:       testutils.NewMockOutputWriter(),

v2/pkg/tmplexec/multiproto/multi_test.go

@@ -23,7 +23,7 @@ var executerOpts protocols.ExecutorOptions
 func setup() {
 	options := testutils.DefaultOptions
 	testutils.Init(options)
-	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
+	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	executerOpts = protocols.ExecutorOptions{
 		Output:       testutils.NewMockOutputWriter(),

v2/pkg/types/types.go

@@ -215,7 +215,7 @@ type Options struct {
 	SystemResolvers bool
 	// ShowActions displays a list of all headless actions
 	ShowActions bool
-	// Metrics enables display of metrics via an http endpoint
+	// Deprecated: Enabled by default through clistats . Metrics enables display of metrics via an http endpoint
 	Metrics bool
 	// Debug mode allows debugging request/responses for the engine
 	Debug bool