cloud templates targets sync (#2959)

* Add s3 bucket template provider

- Refactor the custom github template code
- add interface for template provider

* Validate if aws creds are passed if bucket flag

- refactor s3 provider struct to take client
- add function which returns the aws s3 client
- update error messages

* Add aws s3 bucket flags documentation in README.md

- Rename the github_test.go to customTemplate_test.go

* go mod update

* Move template provider code to pkg/external/customtemplates dir

* Added initial data_source sync to cloud

* Misc

* Add pagination to scan output and scan list (#2858)

* Add pagination to scan output and scan list

* Use time based parameters instead of page numbers

* Fix linting errors

* Do not check limits at client, check at server

* Remove unused constant

* Misc update

* Removed unnecessary flags

* Misc

* Misc

* Misc endpoint additions

* Added more routes

* Typo fix

* Misc fixes

* Misc

* Misc fixes to cloud target logic + use int for IDs

* Misc

* Misc fixes

* Misc

* Misc fixes

* readme update

* Add JSON output support for list-scan option (#2876)

* Add JSON output support for list-scan option

* Fix typo in cloud JSON output description

* Following changes

- Update status(finished, running) to be lower-case by default
- Convert status to upper-case in DisplayScanList()

* Update status to be lower-case by default

* Remove additional json flag, instead use existing

* Merge conflict

* Accomodate comment changes and restructure code

Co-authored-by: Jaideep K <jaideep@one2n.in>

* Use integer IDs for scan tasks

* Added get-templates-targets endpoint + JSON + validation

* Added target count list

* misc option / description updates

* Added changes as per code review

* duplicate options + typo updates

* Added tablewriter for tabular data writing by default

* Fixed list scan endpoint

* Review changes

* workflow fix

* Added cloud tags etc based filtering (#3070)

* Added omitempty for filtering request

* go mod tidy

* misc format update

Co-authored-by: shubhamrasal <shubhamdharmarasal@gmail.com>
Co-authored-by: Ice3man <nizamulrana@gmail.com>
Co-authored-by: Jaideep Khandelwal <jdk2588@gmail.com>
Co-authored-by: Siddharth Shashikar <60960197+shashikarsiddharth@users.noreply.github.com>
Co-authored-by: Jaideep K <jaideep@one2n.in>

.github/workflows/publish-docs.yaml

@@ -12,12 +12,10 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
 
       - name: "Set up Go"
         uses: actions/setup-go@v3
-        with:
+        with: 
           go-version: 1.18
 
       - name: Generate YAML Syntax Documentation
@@ -28,7 +26,7 @@ jobs:
             go install github.com/projectdiscovery/yamldoc-go/cmd/docgen/dstdocgen@main
           fi
           go generate pkg/templates/templates.go
-          go build -o "cmd/docgen/docgen" cmd/docgen/docgen.go
+          go build -o "cmd/docgen/docgen" cmd/docgen/docgen.go 
           ./cmd/docgen/docgen ../SYNTAX-REFERENCE.md ../nuclei-jsonschema.json
           git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT
         working-directory: v2
@@ -38,6 +36,7 @@ jobs:
         run: |
           git config --local user.email "action@github.com"
           git config --local user.name "GitHub Action"
+          git pull
           git add SYNTAX-REFERENCE.md nuclei-jsonschema.json
           git commit -m "Auto Generate Syntax Docs + JSONSchema [$(date)] :robot:" -a
 
@@ -46,4 +45,4 @@ jobs:
         uses: ad-m/github-push-action@master
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          branch: ${{ github.ref }}
+          branch: ${{ github.head_ref }}

README.md

@@ -239,10 +239,10 @@ DEBUG:
    -hc, -health-check        run diagnostic check up
 
 UPDATE:
-   -un, -update                      update nuclei engine to the latest released version
+   -un, -update                          update nuclei engine to the latest released version
-   -ut, -update-templates            update nuclei-templates to latest released version
+   -ut, -update-templates                update nuclei-templates to latest released version
-   -ud, -update-template-dir string  custom directory to install / update nuclei-templates
+   -ud, -update-template-dir string      custom directory to install / update nuclei-templates
-   -duc, -disable-update-check       disable automatic nuclei/templates update check
+   -duc, -disable-update-check           disable automatic nuclei/templates update check
 
 STATISTICS:
    -stats                    display statistics about the running scan

v2/cmd/nuclei/main.go

@@ -295,12 +295,23 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	flagSet.CreateGroup("cloud", "Cloud",
 		flagSet.BoolVar(&options.Cloud, "cloud", false, "run scan on nuclei cloud"),
-		flagSet.StringVarEnv(&options.CloudURL, "cloud-server", "cs", "https://cloud-dev.nuclei.sh", "NUCLEI_CLOUD_SERVER", "nuclei cloud server to use (NUCLEI_CLOUD_SERVER)"),
+		flagSet.StringVarP(&options.AddDatasource, "add-datasource", "ads", "", "add specified data source (s3,github)"),
-		flagSet.StringVarEnv(&options.CloudAPIKey, "cloud-api-key", "ak", "", "NUCLEI_CLOUD_APIKEY", "api-key for the nuclei cloud server (NUCLEI_CLOUD_APIKEY)"),
+		flagSet.StringVarP(&options.AddTarget, "add-target", "atr", "", "add target(s) to cloud"),
-		flagSet.BoolVarP(&options.ScanList, "list-scan", "ls", false, "list previous cloud scans"),
+		flagSet.StringVarP(&options.AddTemplate, "add-template", "atm", "", "add template(s) to cloud"),
-		flagSet.BoolVarP(&options.NoStore, "no-store", "ns", false, "disable scan/output storage on cloud"),
+		flagSet.BoolVarP(&options.ScanList, "list-scan", "lsn", false, "list previous cloud scans"),
-		flagSet.StringVarP(&options.DeleteScan, "delete-scan", "ds", "", "delete scan/output on cloud by scan id"),
+		flagSet.BoolVarP(&options.ListTargets, "list-target", "ltr", false, "list cloud target by id"),
-		flagSet.StringVarP(&options.ScanOutput, "scan-output", "so", "", "display scan output by scan id"),
+		flagSet.BoolVarP(&options.ListTemplates, "list-template", "ltm", false, "list cloud template by id"),
+		flagSet.BoolVarP(&options.ListDatasources, "list-datasource", "lds", false, "list cloud datasource by id"),
+		flagSet.StringVarP(&options.DeleteScan, "delete-scan", "dsn", "", "delete cloud scan by id"),
+		flagSet.StringVarP(&options.RemoveTarget, "delete-target", "dtr", "", "delete target(s) from cloud"),
+		flagSet.StringVarP(&options.RemoveTemplate, "delete-template", "dtm", "", "delete template(s) from cloud"),
+		flagSet.StringVarP(&options.RemoveDatasource, "delete-datasource", "dds", "", "delete specified data source"),
+		flagSet.StringVarP(&options.GetTarget, "get-target", "gtr", "", "get target content by id"),
+		flagSet.StringVarP(&options.GetTemplate, "get-template", "gtm", "", "get template content by id"),
+		flagSet.BoolVarP(&options.NoStore, "no-store", "nos", false, "disable scan/output storage on cloud"),
+		flagSet.StringVarP(&options.ScanOutput, "scan-output", "sno", "", "display scan output by scan id"),
+		flagSet.BoolVar(&options.NoTables, "no-tables", false, "do not display pretty-printed tables"),
+		flagSet.IntVar(&options.OutputLimit, "limit", 100, "limit the number of output to display"),
 	)
 
 	_ = flagSet.Parse()

v2/go.mod

@@ -30,7 +30,7 @@ require (
 	github.com/projectdiscovery/interactsh v1.0.6-0.20220827132222-460cc6270053
 	github.com/projectdiscovery/rawhttp v0.1.4
 	github.com/projectdiscovery/retryabledns v1.0.17
-	github.com/projectdiscovery/retryablehttp-go v1.0.5-0.20221202084821-c1a692a64751
+	github.com/projectdiscovery/retryablehttp-go v1.0.6-0.20221206071935-7924d7d34953
 	github.com/projectdiscovery/stringsutil v0.0.2
 	github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6
 	github.com/remeh/sizedwaitgroup v1.0.0
@@ -87,7 +87,14 @@ require (
 )
 
 require (
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bits-and-blooms/bitset v1.3.1 // indirect
+	github.com/bits-and-blooms/bloom/v3 v3.3.1 // indirect
 	github.com/cloudflare/circl v1.1.0 // indirect
 	github.com/dlclark/regexp2 v1.4.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
@@ -111,6 +118,7 @@ require (
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tidwall/rtred v0.1.2 // indirect
 	github.com/tidwall/tinyqueue v0.1.1 // indirect
+	github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
 )
 
@@ -126,8 +134,6 @@ require (
 	github.com/andybalholm/cascadia v1.1.0 // indirect
 	github.com/antchfx/xpath v1.2.1 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
-	github.com/bits-and-blooms/bitset v1.3.1 // indirect
-	github.com/bits-and-blooms/bloom/v3 v3.3.1 // indirect
 	github.com/c4milo/unpackit v0.1.0 // indirect
 	github.com/caddyserver/certmagic v0.16.3 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
@@ -218,16 +224,11 @@ require (
 	github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/alecthomas/chroma v0.10.0
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.20 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.27 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.20 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.11.26 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.9 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.17.6 // indirect
@@ -250,6 +251,5 @@ require (
 	github.com/src-d/gcfg v1.4.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
-	github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 )

v2/go.sum

@@ -637,8 +637,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
 github.com/projectdiscovery/retryabledns v1.0.17 h1:XKzI26UKYt2g7YLJ/EcyYmM04sfD1vurETecPEpeA1w=
 github.com/projectdiscovery/retryabledns v1.0.17/go.mod h1:Dyhq/f0sGmXueso0+Ah3LbJfsX4PXpBrpfiyjZZ8SDk=
-github.com/projectdiscovery/retryablehttp-go v1.0.5-0.20221202084821-c1a692a64751 h1:QEmZ0E6GDzlTbVE6ty7fCuKR7muWrLqMfQ07VTu6Bd0=
+github.com/projectdiscovery/retryablehttp-go v1.0.6-0.20221206071935-7924d7d34953 h1:8voTfrLQKt/nElO7zx6yoaFZDCeDq2xFHkWEl3fXBBI=
-github.com/projectdiscovery/retryablehttp-go v1.0.5-0.20221202084821-c1a692a64751/go.mod h1:B/xfvUmiJBeq+1kT7AMYL6B/IuPgbyKB7QCKPSfMByc=
+github.com/projectdiscovery/retryablehttp-go v1.0.6-0.20221206071935-7924d7d34953/go.mod h1:Z9FoXiCxITq+Wt6VzqJAu8xA/xDTVrgBFaDk+Xsfgw4=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
 github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
 github.com/projectdiscovery/sliceutil v0.0.1 h1:YoCqCMcdwz+gqNfW5hFY8UvNHoA6SfyBSNkVahatleg=

v2/internal/runner/cloud.go

@@ -0,0 +1,319 @@
+package runner
+
+import (
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/olekukonko/tablewriter"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/internal/runner/nucleicloud"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+)
+
+// Get all the scan lists for a user/apikey.
+func (r *Runner) getScanList(limit int) error {
+	lastTime := "2099-01-02 15:04:05 +0000 UTC"
+	header := []string{"ID", "Timestamp", "Targets", "Templates", "Matched", "Duration", "Status"}
+
+	var (
+		values [][]string
+		count  int
+	)
+	for {
+		items, err := r.cloudClient.GetScans(limit, lastTime)
+		if err != nil {
+			return err
+		}
+		if len(items) == 0 {
+			break
+		}
+		for _, v := range items {
+			count++
+			lastTime = v.CreatedAt.String()
+			res := nucleicloud.PrepareScanListOutput(v)
+			if r.options.JSON {
+				_ = jsoniter.NewEncoder(os.Stdout).Encode(res)
+			} else if !r.options.NoTables {
+				values = append(values, []string{strconv.FormatInt(res.ScanID, 10), res.Timestamp, strconv.Itoa(res.Target), strconv.Itoa(res.Template), strconv.Itoa(res.ScanResult), res.ScanTime, res.ScanStatus})
+			} else {
+				gologger.Silent().Msgf("%d. [%s] [TARGETS: %d] [TEMPLATES: %d] [MATCHED: %d] [DURATION: %s] [STATUS: %s]\n", res.ScanID, res.Timestamp, res.Target, res.Template, res.ScanResult, res.ScanTime, strings.ToUpper(res.ScanStatus))
+
+			}
+		}
+	}
+	if count == 0 {
+		return errors.New("no scan found")
+	}
+	if !r.options.NoTables {
+		r.prettyPrintTable(header, values)
+	}
+	return nil
+}
+
+func (r *Runner) listDatasources() error {
+	datasources, err := r.cloudClient.ListDatasources()
+	if err != nil {
+		return err
+	}
+	if len(datasources) == 0 {
+		return errors.New("no cloud datasource found")
+	}
+
+	header := []string{"ID", "UpdatedAt", "Type", "Repo", "Path"}
+	var values [][]string
+	for _, source := range datasources {
+		if r.options.JSON {
+			_ = jsoniter.NewEncoder(os.Stdout).Encode(source)
+		} else if !r.options.NoTables {
+			values = append(values, []string{strconv.FormatInt(source.ID, 10), source.Updatedat.Format(nucleicloud.DDMMYYYYhhmmss), source.Type, source.Repo, source.Path})
+		} else {
+			gologger.Silent().Msgf("%d. [%s] [%s] [%s] %s", source.ID, source.Updatedat.Format(nucleicloud.DDMMYYYYhhmmss), source.Type, source.Repo, source.Path)
+		}
+	}
+	if !r.options.NoTables {
+		r.prettyPrintTable(header, values)
+	}
+	return nil
+}
+
+func (r *Runner) listTargets() error {
+	items, err := r.cloudClient.ListTargets("")
+	if err != nil {
+		return err
+	}
+	if len(items) == 0 {
+		return errors.New("no target found")
+	}
+
+	header := []string{"ID", "Reference", "Count"}
+	var values [][]string
+	for _, source := range items {
+		if r.options.JSON {
+			_ = jsoniter.NewEncoder(os.Stdout).Encode(source)
+		} else if !r.options.NoTables {
+			values = append(values, []string{strconv.FormatInt(source.ID, 10), source.Reference, strconv.FormatInt(source.Count, 10)})
+		} else {
+			gologger.Silent().Msgf("%d. %s (%d)", source.ID, source.Reference, source.Count)
+		}
+	}
+	if !r.options.NoTables {
+		r.prettyPrintTable(header, values)
+	}
+	return nil
+}
+
+func (r *Runner) listTemplates() error {
+	items, err := r.cloudClient.ListTemplates("")
+	if err != nil {
+		return err
+	}
+	if len(items) == 0 {
+		return errors.New("no template found")
+	}
+
+	header := []string{"ID", "Reference"}
+	var values [][]string
+	for _, source := range items {
+		if r.options.JSON {
+			_ = jsoniter.NewEncoder(os.Stdout).Encode(source)
+		} else if !r.options.NoTables {
+			values = append(values, []string{strconv.FormatInt(source.ID, 10), source.Reference})
+		} else {
+			gologger.Silent().Msgf("%d. %s", source.ID, source.Reference)
+		}
+	}
+	if !r.options.NoTables {
+		r.prettyPrintTable(header, values)
+	}
+	return nil
+}
+
+func (r *Runner) prettyPrintTable(header []string, values [][]string) {
+	writer := tablewriter.NewWriter(os.Stdout)
+	writer.SetHeader(header)
+	writer.AppendBulk(values)
+	writer.Render()
+}
+
+func (r *Runner) deleteScan(id string) error {
+	ID, _ := strconv.ParseInt(id, 10, 64)
+	deleted, err := r.cloudClient.DeleteScan(ID)
+	if !deleted.OK {
+		gologger.Error().Msgf("Error in deleting the scan %s.", id)
+	} else {
+		gologger.Info().Msgf("Scan deleted %s.", id)
+	}
+	return err
+}
+
+func (r *Runner) getResults(id string, limit int) error {
+	ID, _ := strconv.ParseInt(id, 10, 64)
+	err := r.cloudClient.GetResults(ID, false, limit, func(re *output.ResultEvent) {
+		if outputErr := r.output.Write(re); outputErr != nil {
+			gologger.Warning().Msgf("Could not write output: %s", outputErr)
+		}
+	})
+	return err
+}
+
+func (r *Runner) getTarget(id string) error {
+	ID, _ := strconv.ParseInt(id, 10, 64)
+	reader, err := r.cloudClient.GetTarget(ID)
+	if err != nil {
+		return errors.Wrap(err, "could not get target")
+	}
+	defer reader.Close()
+
+	_, _ = io.Copy(os.Stdout, reader)
+	return nil
+}
+
+func (r *Runner) getTemplate(id string) error {
+	ID, _ := strconv.ParseInt(id, 10, 64)
+	reader, err := r.cloudClient.GetTemplate(ID)
+	if err != nil {
+		return errors.Wrap(err, "could not get template")
+	}
+	defer reader.Close()
+
+	_, _ = io.Copy(os.Stdout, reader)
+	return nil
+}
+
+func (r *Runner) removeDatasource(datasource string) error {
+	var source string
+	ID, parseErr := strconv.ParseInt(datasource, 10, 64)
+	if parseErr != nil {
+		source = datasource
+	}
+
+	err := r.cloudClient.RemoveDatasource(ID, source)
+	if err == nil {
+		gologger.Info().Msgf("Datasource deleted %s", datasource)
+	}
+	return err
+}
+
+func (r *Runner) addTemplate(location string) error {
+	walkErr := filepath.WalkDir(location, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".yaml") {
+			return nil
+		}
+		base := filepath.Base(path)
+		reference, templateErr := r.cloudClient.AddTemplate(base, path)
+		if templateErr != nil {
+			gologger.Error().Msgf("Could not upload %s: %s", path, templateErr)
+		} else if reference != "" {
+			gologger.Info().Msgf("Uploaded template %s: %s", base, reference)
+		}
+		return nil
+	})
+	return walkErr
+}
+
+func (r *Runner) addTarget(location string) error {
+	walkErr := filepath.WalkDir(location, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".txt") {
+			return nil
+		}
+		base := filepath.Base(location)
+		reference, targetErr := r.cloudClient.AddTarget(base, location)
+		if targetErr != nil {
+			gologger.Error().Msgf("Could not upload %s: %s", location, targetErr)
+		} else if reference != "" {
+			gologger.Info().Msgf("Uploaded target %s: %s", base, reference)
+		}
+		return nil
+	})
+	return walkErr
+}
+
+func (r *Runner) removeTarget(item string) error {
+	response, err := r.cloudClient.ListTargets(item)
+	if err != nil {
+		return errors.Wrap(err, "could not list targets")
+	}
+	for _, item := range response {
+		if err := r.cloudClient.RemoveTarget(item.ID); err != nil {
+			gologger.Error().Msgf("Error in deleting target %s: %s", item.Reference, err)
+		} else {
+			gologger.Info().Msgf("Target deleted %s", item.Reference)
+		}
+	}
+	return nil
+}
+
+func (r *Runner) removeTemplate(item string) error {
+	response, err := r.cloudClient.ListTemplates(item)
+	if err != nil {
+		return errors.Wrap(err, "could not list templates")
+	}
+	for _, item := range response {
+		if err := r.cloudClient.RemoveTemplate(item.ID); err != nil {
+			gologger.Error().Msgf("Error in deleting template %s: %s", item.Reference, err)
+		} else {
+			gologger.Info().Msgf("Template deleted %s", item.Reference)
+		}
+	}
+	return nil
+}
+
+// initializeCloudDataSources initializes cloud data sources
+func (r *Runner) addCloudDataSource(source string) error {
+	switch source {
+	case "s3":
+		token := strings.Join([]string{r.options.AwsAccessKey, r.options.AwsSecretKey, r.options.AwsRegion}, ":")
+		if _, err := r.processDataSourceItem(r.options.AwsBucketName, token, "s3"); err != nil {
+			return err
+		}
+	case "github":
+		for _, repo := range r.options.GithubTemplateRepo {
+			if _, err := r.processDataSourceItem(repo, r.options.GithubToken, "github"); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (r *Runner) processDataSourceItem(repo, token, Type string) (int64, error) {
+	ID, err := r.cloudClient.StatusDataSource(nucleicloud.StatusDataSourceRequest{Repo: repo, Token: token})
+	if err != nil {
+		if !strings.Contains(err.Error(), "no rows in result set") {
+			return 0, errors.Wrap(err, "could not get data source status")
+		}
+
+		gologger.Info().Msgf("Adding new data source + syncing: %s\n", repo)
+		resp, err := r.cloudClient.AddDataSource(nucleicloud.AddDataSourceRequest{Type: Type, Repo: repo, Token: token})
+		if err != nil {
+			return 0, errors.Wrap(err, "could not add data source")
+		}
+		ID = resp.ID
+		if err = r.cloudClient.SyncDataSource(resp.ID); err != nil {
+			return 0, errors.Wrap(err, "could not sync data source")
+		}
+		if resp.Secret != "" {
+			gologger.Info().Msgf("Webhook URL for added source: %s/datasources/%s/webhook", r.options.CloudURL, resp.Hash)
+			gologger.Info().Msgf("Secret for webhook: %s", resp.Secret)
+		}
+	}
+	if r.options.UpdateTemplates {
+		gologger.Info().Msgf("Syncing data source: %s (%d)\n", repo, ID)
+		if err = r.cloudClient.SyncDataSource(ID); err != nil {
+			return 0, errors.Wrap(err, "could not sync data source")
+		}
+	}
+	return ID, nil
+}

v2/internal/runner/enumerate.go

@@ -20,11 +20,10 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"go.uber.org/atomic"
 )
 
-const DDMMYYYYhhmmss = "2006-01-02 15:04:05"
-
 // runStandardEnumeration runs standard enumeration
 func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions, store *loader.Store, engine *core.Engine) (*atomic.Bool, error) {
 	if r.options.AutomaticScan {
@@ -33,52 +32,12 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions,
 	return r.executeTemplatesInput(store, engine)
 }
 
-// Get all the scan lists for a user/apikey.
-func (r *Runner) getScanList() error {
-	items, err := r.cloudClient.GetScans()
-	loc, _ := time.LoadLocation("Local")
-
-	for _, v := range items {
-		status := "FINISHED"
-		t := v.FinishedAt
-		duration := t.Sub(v.CreatedAt)
-		if !v.Finished {
-			status = "RUNNING"
-			t = time.Now().UTC()
-			duration = t.Sub(v.CreatedAt).Round(60 * time.Second)
-		}
-
-		val := v.CreatedAt.In(loc).Format(DDMMYYYYhhmmss)
-
-		gologger.Silent().Msgf("%s [%s] [STATUS: %s] [MATCHED: %d] [TARGETS: %d] [TEMPLATES: %d] [DURATION: %s]\n", v.Id, val, status, v.Matches, v.Targets, v.Templates, duration)
-	}
-	return err
-}
-
-func (r *Runner) deleteScan(id string) error {
-	deleted, err := r.cloudClient.DeleteScan(id)
-	if !deleted.OK {
-		gologger.Info().Msgf("Error in deleting the scan %s.", id)
-	} else {
-		gologger.Info().Msgf("Scan deleted %s.", id)
-	}
-	return err
-}
-
-func (r *Runner) getResults(id string) error {
-	err := r.cloudClient.GetResults(id, func(re *output.ResultEvent) {
-		if outputErr := r.output.Write(re); outputErr != nil {
-			gologger.Warning().Msgf("Could not write output: %s", outputErr)
-		}
-	}, false)
-	return err
-}
-
 // runCloudEnumeration runs cloud based enumeration
-func (r *Runner) runCloudEnumeration(store *loader.Store, nostore bool) (*atomic.Bool, error) {
+func (r *Runner) runCloudEnumeration(store *loader.Store, cloudTemplates, cloudTargets []string, nostore bool, limit int) (*atomic.Bool, error) {
+	count := &atomic.Int64{}
 	now := time.Now()
 	defer func() {
-		gologger.Info().Msgf("Scan execution took %s", time.Since(now))
+		gologger.Info().Msgf("Scan execution took %s and found %d results", time.Since(now), count.Load())
 	}()
 	results := &atomic.Bool{}
 
@@ -110,17 +69,24 @@ func (r *Runner) runCloudEnumeration(store *loader.Store, nostore bool) (*atomic
 	taskID, err := r.cloudClient.AddScan(&nucleicloud.AddScanRequest{
 		RawTargets:       targets,
 		PublicTemplates:  templates,
+		CloudTargets:     cloudTargets,
+		CloudTemplates:   cloudTemplates,
 		PrivateTemplates: privateTemplates,
 		IsTemporary:      nostore,
+		Filtering:        getCloudFilteringFromOptions(r.options),
 	})
 	if err != nil {
 		return results, err
 	}
-	gologger.Info().Msgf("Created task with ID: %s", taskID)
+	gologger.Info().Msgf("Created task with ID: %d", taskID)
+	if nostore {
+		gologger.Info().Msgf("Cloud scan storage: disabled")
+	}
 	time.Sleep(3 * time.Second)
 
-	err = r.cloudClient.GetResults(taskID, func(re *output.ResultEvent) {
+	err = r.cloudClient.GetResults(taskID, true, limit, func(re *output.ResultEvent) {
 		results.CompareAndSwap(false, true)
+		_ = count.Inc()
 
 		if outputErr := r.output.Write(re); outputErr != nil {
 			gologger.Warning().Msgf("Could not write output: %s", err)
@@ -130,7 +96,7 @@ func (r *Runner) runCloudEnumeration(store *loader.Store, nostore bool) (*atomic
 				gologger.Warning().Msgf("Could not create issue on tracker: %s", err)
 			}
 		}
-	}, true)
+	})
 	return results, err
 }
 
@@ -150,3 +116,22 @@ func gzipBase64EncodeData(data []byte) string {
 	encoded := base64.StdEncoding.EncodeToString(buf.Bytes())
 	return encoded
 }
+
+func getCloudFilteringFromOptions(options *types.Options) *nucleicloud.AddScanRequestConfiguration {
+	return &nucleicloud.AddScanRequestConfiguration{
+		Authors:           options.Authors,
+		Tags:              options.Tags,
+		ExcludeTags:       options.ExcludeTags,
+		IncludeTags:       options.IncludeTags,
+		IncludeIds:        options.IncludeIds,
+		ExcludeIds:        options.ExcludeIds,
+		IncludeTemplates:  options.IncludeTemplates,
+		ExcludedTemplates: options.ExcludedTemplates,
+		ExcludeMatchers:   options.ExcludeMatchers,
+		Severities:        options.Severities,
+		ExcludeSeverities: options.ExcludeSeverities,
+		Protocols:         options.Protocols,
+		ExcludeProtocols:  options.ExcludeProtocols,
+		IncludeConditions: options.IncludeConditions,
+	}
+}

v2/internal/runner/nucleicloud/cloud.go

@@ -3,9 +3,15 @@ package nucleicloud
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
+	"mime/multipart"
 	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -23,9 +29,9 @@ type Client struct {
 }
 
 const (
-	pollInterval   = 1 * time.Second
+	pollInterval   = 3 * time.Second
-	defaultBaseURL = "http://webapp.localhost"
 	resultSize     = 100
+	defaultBaseURL = "https://cloud-dev.nuclei.sh"
 )
 
 // HTTPErrorRetryPolicy is to retry for HTTPCodes >= 500.
@@ -41,7 +47,8 @@ func HTTPErrorRetryPolicy() func(ctx context.Context, resp *http.Response, err e
 // New returns a nuclei-cloud API client
 func New(baseURL, apiKey string) *Client {
 	options := retryablehttp.DefaultOptionsSingle
-	options.Timeout = 15 * time.Second
+	options.NoAdjustTimeout = true
+	options.Timeout = 60 * time.Second
 	options.CheckRetry = HTTPErrorRetryPolicy()
 	client := retryablehttp.NewClient(options)
 
@@ -53,57 +60,47 @@ func New(baseURL, apiKey string) *Client {
 }
 
 // AddScan adds a scan for templates and target to nuclei server
-func (c *Client) AddScan(req *AddScanRequest) (string, error) {
+func (c *Client) AddScan(req *AddScanRequest) (int64, error) {
 	var buf bytes.Buffer
 	if err := jsoniter.NewEncoder(&buf).Encode(req); err != nil {
-		return "", errors.Wrap(err, "could not json encode scan request")
+		return 0, errors.Wrap(err, "could not encode request")
 	}
 	httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/scan", c.baseURL), bytes.NewReader(buf.Bytes()))
 	if err != nil {
-		return "", errors.Wrap(err, "could not make request")
+		return 0, errors.Wrap(err, "could not make request")
 	}
-	httpReq.Header.Set("X-API-Key", c.apiKey)
 
-	resp, err := c.httpclient.Do(httpReq)
+	resp, err := c.sendRequest(httpReq)
 	if err != nil {
-		return "", errors.Wrap(err, "could not do add scan request")
+		return 0, errors.Wrap(err, "could not do request")
 	}
-	if resp.StatusCode != 200 {
+	defer resp.Body.Close()
-		data, _ := io.ReadAll(resp.Body)
+
-		resp.Body.Close()
+	var data map[string]int64
-		return "", errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data))
-	}
-	var data map[string]string
 	if err := jsoniter.NewDecoder(resp.Body).Decode(&data); err != nil {
-		resp.Body.Close()
+		return 0, errors.Wrap(err, "could not decode resp")
-		return "", errors.Wrap(err, "could not decode resp")
 	}
-	resp.Body.Close()
 	id := data["id"]
 	return id, nil
 }
 
 // GetResults gets results from nuclei server for an ID
 // until there are no more results left to retrieve.
-func (c *Client) GetResults(ID string, callback func(*output.ResultEvent), checkProgress bool) error {
+func (c *Client) GetResults(ID int64, checkProgress bool, limit int, callback func(*output.ResultEvent)) error {
 	lastID := int64(0)
+
 	for {
-		uri := fmt.Sprintf("%s/results?id=%s&from=%d&size=%d", c.baseURL, ID, lastID, resultSize)
+		uri := fmt.Sprintf("%s/results?id=%d&from=%d&size=%d", c.baseURL, ID, lastID, limit)
 		httpReq, err := retryablehttp.NewRequest(http.MethodGet, uri, nil)
 		if err != nil {
 			return errors.Wrap(err, "could not make request")
 		}
-		httpReq.Header.Set("X-API-Key", c.apiKey)
 
-		resp, err := c.httpclient.Do(httpReq)
+		resp, err := c.sendRequest(httpReq)
 		if err != nil {
-			return errors.Wrap(err, "could not do ger result request")
+			return errors.Wrap(err, "could not do request")
-		}
-		if resp.StatusCode != 200 {
-			data, _ := io.ReadAll(resp.Body)
-			resp.Body.Close()
-			return errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data))
 		}
+
 		var items GetResultsResponse
 		if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil {
 			resp.Body.Close()
@@ -135,61 +132,409 @@ func (c *Client) GetResults(ID string, callback func(*output.ResultEvent), check
 	return nil
 }
 
-func (c *Client) GetScans() ([]GetScanRequest, error) {
+func (c *Client) GetScans(limit int, from string) ([]GetScanRequest, error) {
 	var items []GetScanRequest
-	httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/scan", c.baseURL), nil)
+	httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/scan?from=%s&size=%d", c.baseURL, url.QueryEscape(from), limit), nil)
 	if err != nil {
 		return items, errors.Wrap(err, "could not make request")
 	}
-	httpReq.Header.Set("X-API-Key", c.apiKey)
 
-	resp, err := c.httpclient.Do(httpReq)
+	resp, err := c.sendRequest(httpReq)
 	if err != nil {
-		return items, errors.Wrap(err, "could not make request.")
+		return nil, errors.Wrap(err, "could not do request")
-	}
-	if err != nil {
-		return items, errors.Wrap(err, "could not do get response.")
-	}
-	if resp.StatusCode != 200 {
-		data, _ := io.ReadAll(resp.Body)
-		resp.Body.Close()
-		return items, errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data))
 	}
+	defer resp.Body.Close()
+
 	if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil {
-		resp.Body.Close()
 		return items, errors.Wrap(err, "could not decode results")
 	}
-	resp.Body.Close()
-
 	return items, nil
 }
 
 // Delete a scan and it's issues by the scan id.
-func (c *Client) DeleteScan(id string) (DeleteScanResults, error) {
+func (c *Client) DeleteScan(id int64) (DeleteScanResults, error) {
 	deletescan := DeleteScanResults{}
-	httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/scan?id=%s", c.baseURL, id), nil)
+	httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/scan?id=%d", c.baseURL, id), nil)
 	if err != nil {
 		return deletescan, errors.Wrap(err, "could not make request")
 	}
-	httpReq.Header.Set("X-API-Key", c.apiKey)
 
-	resp, err := c.httpclient.Do(httpReq)
+	resp, err := c.sendRequest(httpReq)
 	if err != nil {
-		return deletescan, errors.Wrap(err, "could not make request")
+		return deletescan, errors.Wrap(err, "could not do request")
-	}
-	if err != nil {
-		return deletescan, errors.Wrap(err, "could not do get result request")
-	}
-	if resp.StatusCode != 200 {
-		data, _ := io.ReadAll(resp.Body)
-		resp.Body.Close()
-		return deletescan, errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data))
 	}
+	defer resp.Body.Close()
+
 	if err := jsoniter.NewDecoder(resp.Body).Decode(&deletescan); err != nil {
-		resp.Body.Close()
 		return deletescan, errors.Wrap(err, "could not delete scan")
 	}
-	resp.Body.Close()
-
 	return deletescan, nil
 }
+
+// StatusDataSource returns the status for a data source
+func (c *Client) StatusDataSource(statusRequest StatusDataSourceRequest) (int64, error) {
+	var buf bytes.Buffer
+	if err := jsoniter.NewEncoder(&buf).Encode(statusRequest); err != nil {
+		return 0, errors.Wrap(err, "could not encode request")
+	}
+	httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/datasources/status", c.baseURL), bytes.NewReader(buf.Bytes()))
+	if err != nil {
+		return 0, errors.Wrap(err, "could not make request")
+	}
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return 0, errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+
+	var data StatusDataSourceResponse
+	if err := jsoniter.NewDecoder(resp.Body).Decode(&data); err != nil {
+		return 0, errors.Wrap(err, "could not decode resp")
+	}
+	return data.ID, nil
+}
+
+// AddDataSource adds a new data source
+func (c *Client) AddDataSource(req AddDataSourceRequest) (*AddDataSourceResponse, error) {
+	var buf bytes.Buffer
+	if err := jsoniter.NewEncoder(&buf).Encode(req); err != nil {
+		return nil, errors.Wrap(err, "could not encode request")
+	}
+	httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/datasources", c.baseURL), bytes.NewReader(buf.Bytes()))
+	if err != nil {
+		return nil, errors.Wrap(err, "could not make request")
+	}
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+
+	var data AddDataSourceResponse
+	if err := jsoniter.NewDecoder(resp.Body).Decode(&data); err != nil {
+		return nil, errors.Wrap(err, "could not decode resp")
+	}
+	return &data, nil
+}
+
+// SyncDataSource syncs contents for a data source. The call blocks until
+// update is completed.
+func (c *Client) SyncDataSource(ID int64) error {
+	httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/datasources/%d/sync", c.baseURL, ID), nil)
+	if err != nil {
+		return errors.Wrap(err, "could not make request")
+	}
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+	_, _ = io.Copy(io.Discard, resp.Body)
+	return nil
+}
+
+// ExistsDataSourceItem identifies whether data source item exist
+func (c *Client) ExistsDataSourceItem(req ExistsDataSourceItemRequest) error {
+	var buf bytes.Buffer
+	if err := jsoniter.NewEncoder(&buf).Encode(req); err != nil {
+		return errors.Wrap(err, "could not encode request")
+	}
+	httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/datasources/exists", c.baseURL), bytes.NewReader(buf.Bytes()))
+	if err != nil {
+		return errors.Wrap(err, "could not make request")
+	}
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+	_, _ = io.Copy(io.Discard, resp.Body)
+	return nil
+}
+
+func (c *Client) ListDatasources() ([]GetDataSourceResponse, error) {
+	var items []GetDataSourceResponse
+	httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/datasources", c.baseURL), nil)
+	if err != nil {
+		return items, errors.Wrap(err, "could not make request")
+	}
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+
+	if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil {
+		return items, errors.Wrap(err, "could not decode results")
+	}
+	return items, nil
+}
+
+func (c *Client) ListTargets(query string) ([]GetTargetResponse, error) {
+	var builder strings.Builder
+	_, _ = builder.WriteString(c.baseURL)
+	_, _ = builder.WriteString("/targets")
+	if query != "" {
+		_, _ = builder.WriteString("?query=")
+		_, _ = builder.WriteString(url.QueryEscape(query))
+	}
+
+	var items []GetTargetResponse
+	httpReq, err := retryablehttp.NewRequest(http.MethodGet, builder.String(), nil)
+	if err != nil {
+		return items, errors.Wrap(err, "could not make request")
+	}
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+
+	if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil {
+		return items, errors.Wrap(err, "could not decode results")
+	}
+	return items, nil
+}
+
+func (c *Client) ListTemplates(query string) ([]GetTemplatesResponse, error) {
+	var builder strings.Builder
+	_, _ = builder.WriteString(c.baseURL)
+	_, _ = builder.WriteString("/templates")
+	if query != "" {
+		_, _ = builder.WriteString("?query=")
+		_, _ = builder.WriteString(url.QueryEscape(query))
+	}
+
+	var items []GetTemplatesResponse
+	httpReq, err := retryablehttp.NewRequest(http.MethodGet, builder.String(), nil)
+	if err != nil {
+		return items, errors.Wrap(err, "could not make request")
+	}
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+
+	if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil {
+		return items, errors.Wrap(err, "could not decode results")
+	}
+	return items, nil
+}
+
+func (c *Client) RemoveDatasource(datasource int64, name string) error {
+	var builder strings.Builder
+	_, _ = builder.WriteString(c.baseURL)
+	_, _ = builder.WriteString("/datasources")
+
+	if name != "" {
+		_, _ = builder.WriteString("?name=")
+		_, _ = builder.WriteString(name)
+	} else if datasource != 0 {
+		_, _ = builder.WriteString("?id=")
+		_, _ = builder.WriteString(strconv.FormatInt(datasource, 10))
+	}
+
+	httpReq, err := retryablehttp.NewRequest(http.MethodDelete, builder.String(), nil)
+	if err != nil {
+		return errors.Wrap(err, "could not make request")
+	}
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+	_, _ = io.Copy(io.Discard, resp.Body)
+	return nil
+}
+
+func (c *Client) AddTemplate(name, contents string) (string, error) {
+	file, err := os.Open(contents)
+	if err != nil {
+		return "", errors.Wrap(err, "could not open contents")
+	}
+	defer file.Close()
+
+	var buf bytes.Buffer
+	writer := multipart.NewWriter(&buf)
+	_ = writer.WriteField("name", name)
+	fileWriter, _ := writer.CreateFormFile("file", filepath.Base(contents))
+	_, _ = io.Copy(fileWriter, file)
+	_ = writer.Close()
+
+	httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/templates", c.baseURL), &buf)
+	if err != nil {
+		return "", errors.Wrap(err, "could not make request")
+	}
+	httpReq.Header.Set("Content-Type", writer.FormDataContentType())
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return "", errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+
+	var item AddItemResponse
+	if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil {
+		return "", errors.Wrap(err, "could not decode results")
+	}
+	return item.Ok, nil
+}
+
+func (c *Client) AddTarget(name, contents string) (string, error) {
+	file, err := os.Open(contents)
+	if err != nil {
+		return "", errors.Wrap(err, "could not open contents")
+	}
+	defer file.Close()
+
+	var buf bytes.Buffer
+	writer := multipart.NewWriter(&buf)
+	_ = writer.WriteField("name", name)
+	fileWriter, _ := writer.CreateFormFile("file", filepath.Base(contents))
+	_, _ = io.Copy(fileWriter, file)
+	_ = writer.Close()
+
+	httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/targets", c.baseURL), &buf)
+	if err != nil {
+		return "", errors.Wrap(err, "could not make request")
+	}
+	httpReq.Header.Set("Content-Type", writer.FormDataContentType())
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return "", errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+
+	var item AddItemResponse
+	if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil {
+		return "", errors.Wrap(err, "could not decode results")
+	}
+	return item.Ok, nil
+}
+
+func (c *Client) RemoveTemplate(ID int64) error {
+	httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/templates/%d", c.baseURL, ID), nil)
+	if err != nil {
+		return errors.Wrap(err, "could not make request")
+	}
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+	_, _ = io.Copy(io.Discard, resp.Body)
+	return nil
+}
+
+func (c *Client) RemoveTarget(ID int64) error {
+	httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/targets/%d", c.baseURL, ID), nil)
+	if err != nil {
+		return errors.Wrap(err, "could not make request")
+	}
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+	_, _ = io.Copy(io.Discard, resp.Body)
+	return nil
+}
+
+func (c *Client) GetTarget(ID int64) (io.ReadCloser, error) {
+	httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/targets/%d", c.baseURL, ID), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not make request")
+	}
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not do request")
+	}
+	return resp.Body, nil
+}
+
+func (c *Client) GetTemplate(ID int64) (io.ReadCloser, error) {
+	httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/templates/%d", c.baseURL, ID), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not make request")
+	}
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not do request")
+	}
+	return resp.Body, nil
+}
+
+func (c *Client) ExistsTarget(id int64) (ExistsInputResponse, error) {
+	var item ExistsInputResponse
+	httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/targets/%d/exists", c.baseURL, id), nil)
+	if err != nil {
+		return item, errors.Wrap(err, "could not make request")
+	}
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return item, errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+
+	if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil {
+		return item, errors.Wrap(err, "could not decode results")
+	}
+	return item, nil
+}
+
+func (c *Client) ExistsTemplate(id int64) (ExistsInputResponse, error) {
+	var item ExistsInputResponse
+	httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/templates/%d/exists", c.baseURL, id), nil)
+	if err != nil {
+		return item, errors.Wrap(err, "could not make request")
+	}
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return item, errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+
+	if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil {
+		return item, errors.Wrap(err, "could not decode results")
+	}
+	return item, nil
+}
+
+const apiKeyParameter = "X-API-Key"
+
+type errorResponse struct {
+	Message string `json:"message"`
+}
+
+func (c *Client) sendRequest(req *retryablehttp.Request) (*http.Response, error) {
+	req.Header.Set(apiKeyParameter, c.apiKey)
+
+	resp, err := c.httpclient.Do(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not do request")
+	}
+	if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest {
+		data, _ := io.ReadAll(resp.Body)
+		resp.Body.Close()
+		var errRes errorResponse
+		if err = json.NewDecoder(bytes.NewReader(data)).Decode(&errRes); err == nil {
+			return nil, errors.New(errRes.Message)
+		}
+		return nil, fmt.Errorf("unknown error, status code: %d=%s", resp.StatusCode, string(data))
+	}
+	return resp, nil
+}

v2/internal/runner/nucleicloud/types.go

@@ -2,6 +2,9 @@ package nucleicloud
 
 import (
 	"time"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
 )
 
 // AddScanRequest is a nuclei scan input item.
@@ -13,7 +16,32 @@ type AddScanRequest struct {
 	// PrivateTemplates is a map of template-name->contents that
 	// are private to the user executing the scan. (TODO: TBD)
 	PrivateTemplates map[string]string `json:"private_templates,omitempty"`
-	IsTemporary      bool              `json:"is_temporary"`
+	// CloudTargets is a list of cloud targets for the scan
+	CloudTargets []string `json:"cloud_targets,omitempty"`
+	// CloudTemplates is a list of cloud templates for the scan
+	CloudTemplates []string `json:"cloud_templates,omitempty"`
+	// Filtering contains optional filtering options for scan additions
+	Filtering *AddScanRequestConfiguration `json:"filtering"`
+
+	IsTemporary bool `json:"is_temporary"`
+}
+
+// AddScanRequestConfiguration contains filtering options for scan addition
+type AddScanRequestConfiguration struct {
+	Authors           []string            `json:"author,omitempty"`
+	Tags              []string            `json:"tags,omitempty"`
+	ExcludeTags       []string            `json:"exclude-tags,omitempty"`
+	IncludeTags       []string            `json:"include-tags,omitempty"`
+	IncludeIds        []string            `json:"include-ids,omitempty"`
+	ExcludeIds        []string            `json:"exclude-ids,omitempty"`
+	IncludeTemplates  []string            `json:"include-templates,omitempty"`
+	ExcludedTemplates []string            `json:"exclude-templates,omitempty"`
+	ExcludeMatchers   []string            `json:"exclude-matchers,omitempty"`
+	Severities        severity.Severities `json:"severities,omitempty"`
+	ExcludeSeverities severity.Severities `json:"exclude-severities,omitempty"`
+	Protocols         types.ProtocolTypes `json:"protocols,omitempty"`
+	ExcludeProtocols  types.ProtocolTypes `json:"exclude-protocols,omitempty"`
+	IncludeConditions []string            `json:"include-conditions,omitempty"`
 }
 
 type GetResultsResponse struct {
@@ -22,7 +50,7 @@ type GetResultsResponse struct {
 }
 
 type GetScanRequest struct {
-	Id         string    `json:"id"`
+	Id         int64     `json:"id"`
 	Total      int32     `json:"total"`
 	Current    int32     `json:"current"`
 	Finished   bool      `json:"finished"`
@@ -33,6 +61,13 @@ type GetScanRequest struct {
 	Matches    int64     `json:"matches"`
 }
 
+// AddDataSourceResponse is a add data source response item.
+type AddDataSourceResponse struct {
+	ID     int64  `json:"id"`
+	Hash   string `json:"hash"`
+	Secret string `json:"secret,omitempty"`
+}
+
 type GetResultsResponseItem struct {
 	ID  int64  `json:"id"`
 	Raw string `json:"raw"`
@@ -41,3 +76,79 @@ type GetResultsResponseItem struct {
 type DeleteScanResults struct {
 	OK bool `json:"ok"`
 }
+
+// StatusDataSourceRequest is a add data source request item.
+type StatusDataSourceRequest struct {
+	Repo  string `json:"repo"`
+	Token string `json:"token"`
+}
+
+// StatusDataSourceResponse is a add data source response item.
+type StatusDataSourceResponse struct {
+	ID int64 `json:"id"`
+}
+
+// AddDataSourceRequest is a add data source request item.
+type AddDataSourceRequest struct {
+	Type  string `json:"type"`
+	Repo  string `json:"repo"`
+	Token string `json:"token"`
+	Sync  bool   `json:"sync"`
+}
+
+// ExistsDataSourceItemRequest is a request to identify whether a data
+// source item exists.
+type ExistsDataSourceItemRequest struct {
+	Type     string `json:"type"`
+	Contents string `json:"contents"`
+}
+
+// GetDataSourceResponse is response for a get data source request
+type GetDataSourceResponse struct {
+	ID        int64     `json:"id"`
+	Hash      string    `json:"hash"`
+	Type      string    `json:"type"`
+	Path      string    `json:"path"`
+	Repo      string    `json:"repo"`
+	Updatedat time.Time `json:"updated_at"`
+}
+
+// GetTargetResponse is the response for a get target request
+type GetTargetResponse struct {
+	ID         int64  `json:"id"`
+	DataSource int64  `json:"data_source"`
+	Name       string `json:"name"`
+	Reference  string `json:"reference"`
+	Count      int64  `json:"count"`
+	Hash       string `json:"hash"`
+	Type       string `json:"type"`
+}
+
+// GetTemplatesResponse is the response for a get templates request
+type GetTemplatesResponse struct {
+	ID         int64  `json:"id,omitempty"`
+	DataSource int64  `json:"data_source,omitempty"`
+	Name       string `json:"name,omitempty"`
+	Reference  string `json:"reference,omitempty"`
+	Hash       string `json:"hash,omitempty"`
+	Type       string `json:"type,omitempty"`
+}
+
+// AddItemResponse is the response to add item request
+type AddItemResponse struct {
+	Ok string `json:"ok"`
+}
+
+type ListScanOutput struct {
+	Timestamp  string `json:"timestamp"`
+	ScanID     int64  `json:"scan_id"`
+	ScanTime   string `json:"scan_time"`
+	ScanResult int    `json:"scan_result"`
+	ScanStatus string `json:"scan_status"`
+	Target     int    `json:"target"`
+	Template   int    `json:"template"`
+}
+
+type ExistsInputResponse struct {
+	Reference string `json:"reference"`
+}

v2/internal/runner/nucleicloud/utils.go

@@ -5,10 +5,13 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 )
 
+const DDMMYYYYhhmmss = "2006-01-02 15:04:05"
+
 // ReadCatalogChecksum reads catalog checksum from nuclei-templates repository
 func ReadCatalogChecksum() map[string]string {
 	config, _ := config.ReadConfiguration()
@@ -38,3 +41,29 @@ func ReadCatalogChecksum() map[string]string {
 	}
 	return checksums
 }
+
+func PrepareScanListOutput(v GetScanRequest) ListScanOutput {
+	output := ListScanOutput{}
+	loc, _ := time.LoadLocation("Local")
+	status := "finished"
+
+	t := v.FinishedAt
+	duration := t.Sub(v.CreatedAt)
+
+	if !v.Finished {
+		status = "running"
+		t = time.Now().UTC()
+		duration = t.Sub(v.CreatedAt).Round(60 * time.Second)
+	}
+
+	val := v.CreatedAt.In(loc).Format(DDMMYYYYhhmmss)
+
+	output.Timestamp = val
+	output.ScanID = v.Id
+	output.ScanTime = duration.String()
+	output.ScanResult = int(v.Matches)
+	output.ScanStatus = status
+	output.Target = int(v.Targets)
+	output.Template = int(v.Templates)
+	return output
+}

v2/internal/runner/options.go

@@ -160,16 +160,7 @@ func validateOptions(options *types.Options) error {
 	}
 	// Verify aws secrets are passed if s3 template bucket passed
 	if options.AwsBucketName != "" && options.UpdateTemplates {
-		var missing []string
+		missing := validateMissingS3Options(options)
-		if options.AwsAccessKey == "" {
-			missing = append(missing, "AWS_ACCESS_KEY")
-		}
-		if options.AwsSecretKey == "" {
-			missing = append(missing, "AWS_SECRET_KEY")
-		}
-		if options.AwsRegion == "" {
-			missing = append(missing, "AWS_REGION")
-		}
 		if missing != nil {
 			return fmt.Errorf("aws s3 bucket details are missing. Please provide %s", strings.Join(missing, ","))
 		}
@@ -195,9 +186,63 @@ func validateOptions(options *types.Options) error {
 		return errors.New("ipv4 and/or ipv6 must be selected")
 	}
 
+	// Validate cloud option
+	if err := validateCloudOptions(options); err != nil {
+		return err
+	}
 	return nil
 }
 
+func validateCloudOptions(options *types.Options) error {
+	if options.HasCloudOptions() && !options.Cloud {
+		return errors.New("cloud flags cannot be used without cloud option")
+	}
+	if options.Cloud {
+		if options.CloudAPIKey == "" {
+			return errors.New("missing NUCLEI_CLOUD_API env variable")
+		}
+		var missing []string
+		switch options.AddDatasource {
+		case "s3":
+			missing = validateMissingS3Options(options)
+		case "github":
+			missing = validateMissingGithubOptions(options)
+		}
+		if len(missing) > 0 {
+			return fmt.Errorf("missing %v env variables", strings.Join(missing, ", "))
+		}
+	}
+	return nil
+}
+
+func validateMissingS3Options(options *types.Options) []string {
+	var missing []string
+	if options.AwsBucketName == "" {
+		missing = append(missing, "AWS_TEMPLATE_BUCKET")
+	}
+	if options.AwsAccessKey == "" {
+		missing = append(missing, "AWS_ACCESS_KEY")
+	}
+	if options.AwsSecretKey == "" {
+		missing = append(missing, "AWS_SECRET_KEY")
+	}
+	if options.AwsRegion == "" {
+		missing = append(missing, "AWS_REGION")
+	}
+	return missing
+}
+
+func validateMissingGithubOptions(options *types.Options) []string {
+	var missing []string
+	if options.GithubToken == "" {
+		missing = append(missing, "GITHUB_TOKEN")
+	}
+	if len(options.GithubTemplateRepo) == 0 {
+		missing = append(missing, "GITHUB_TEMPLATE_REPO")
+	}
+	return missing
+}
+
 // configureOutput configures the output logging levels to be displayed on the screen
 func configureOutput(options *types.Options) {
 	// If the user desires verbose output, show verbose output
@@ -275,6 +320,14 @@ func validateCertificatePaths(certificatePaths []string) {
 
 // Read the input from env and set options
 func readEnvInputVars(options *types.Options) {
+	if strings.EqualFold(os.Getenv("NUCLEI_CLOUD"), "true") {
+		options.Cloud = true
+	}
+	if options.CloudURL = os.Getenv("NUCLEI_CLOUD_SERVER"); options.CloudURL == "" {
+		options.CloudURL = "https://cloud-dev.nuclei.sh"
+	}
+	options.CloudAPIKey = os.Getenv("NUCLEI_CLOUD_API")
+
 	options.GithubToken = os.Getenv("GITHUB_TOKEN")
 	repolist := os.Getenv("GITHUB_TEMPLATE_REPO")
 	if repolist != "" {

v2/internal/runner/runner.go

@@ -11,6 +11,7 @@ import (
 	_ "net/http/pprof"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -77,6 +78,7 @@ type Runner struct {
 	pprofServer       *http.Server
 	customTemplates   []customtemplates.Provider
 	cloudClient       *nucleicloud.Client
+	cloudTargets      []string
 }
 
 const pprofServerAddress = "127.0.0.1:8086"
@@ -174,12 +176,29 @@ func New(options *types.Options) (*Runner, error) {
 		}()
 	}
 
-	if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates {
+	if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && (options.UpdateTemplates && !options.Cloud) {
 		os.Exit(0)
 	}
 
 	// Initialize the input source
-	hmapInput, err := hybrid.New(options)
+	hmapInput, err := hybrid.New(&hybrid.Options{
+		Options: options,
+		NotFoundCallback: func(target string) bool {
+			parsed, parseErr := strconv.ParseInt(target, 10, 64)
+			if parseErr != nil {
+				if err := runner.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Contents: target, Type: "targets"}); err == nil {
+					runner.cloudTargets = append(runner.cloudTargets, target)
+					return true
+				}
+				return false
+			}
+			if exists, err := runner.cloudClient.ExistsTarget(parsed); err == nil {
+				runner.cloudTargets = append(runner.cloudTargets, exists.Reference)
+				return true
+			}
+			return false
+		},
+	})
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create input provider")
 	}
@@ -401,6 +420,26 @@ func (r *Runner) RunEnumeration() error {
 	if err != nil {
 		return errors.Wrap(err, "could not load templates from config")
 	}
+
+	var cloudTemplates []string
+	if r.options.Cloud {
+		// hook template loading
+		store.NotFoundCallback = func(template string) bool {
+			parsed, parseErr := strconv.ParseInt(template, 10, 64)
+			if parseErr != nil {
+				if err := r.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Type: "templates", Contents: template}); err == nil {
+					cloudTemplates = append(cloudTemplates, template)
+					return true
+				}
+				return false
+			}
+			if exists, err := r.cloudClient.ExistsTemplate(parsed); err == nil {
+				cloudTemplates = append(cloudTemplates, exists.Reference)
+				return true
+			}
+			return false
+		}
+	}
 	if r.options.Validate {
 		if err := store.ValidateTemplates(); err != nil {
 			return err
@@ -443,14 +482,36 @@ func (r *Runner) RunEnumeration() error {
 	var results *atomic.Bool
 	if r.options.Cloud {
 		if r.options.ScanList {
-			err = r.getScanList()
+			err = r.getScanList(r.options.OutputLimit)
 		} else if r.options.DeleteScan != "" {
 			err = r.deleteScan(r.options.DeleteScan)
 		} else if r.options.ScanOutput != "" {
-			err = r.getResults(r.options.ScanOutput)
+			err = r.getResults(r.options.ScanOutput, r.options.OutputLimit)
+		} else if r.options.ListDatasources {
+			err = r.listDatasources()
+		} else if r.options.ListTargets {
+			err = r.listTargets()
+		} else if r.options.ListTemplates {
+			err = r.listTemplates()
+		} else if r.options.AddDatasource != "" {
+			err = r.addCloudDataSource(r.options.AddDatasource)
+		} else if r.options.RemoveDatasource != "" {
+			err = r.removeDatasource(r.options.RemoveDatasource)
+		} else if r.options.AddTarget != "" {
+			err = r.addTarget(r.options.AddTarget)
+		} else if r.options.AddTemplate != "" {
+			err = r.addTemplate(r.options.AddTemplate)
+		} else if r.options.GetTarget != "" {
+			err = r.getTarget(r.options.GetTarget)
+		} else if r.options.GetTemplate != "" {
+			err = r.getTemplate(r.options.GetTemplate)
+		} else if r.options.RemoveTarget != "" {
+			err = r.removeTarget(r.options.RemoveTarget)
+		} else if r.options.RemoveTemplate != "" {
+			err = r.removeTemplate(r.options.RemoveTemplate)
 		} else {
 			gologger.Info().Msgf("Running scan on cloud with URL %s", r.options.CloudURL)
-			results, err = r.runCloudEnumeration(store, r.options.NoStore)
+			results, err = r.runCloudEnumeration(store, cloudTemplates, r.cloudTargets, r.options.NoStore, r.options.OutputLimit)
 			enumeration = true
 		}
 	} else {
@@ -615,8 +676,9 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
 	if len(store.Workflows()) > 0 {
 		gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows()))
 	}
-
+	if r.hmapInputProvider.Count() > 0 {
-	gologger.Info().Msgf("Targets loaded for scan: %d", r.hmapInputProvider.Count())
+		gologger.Info().Msgf("Targets loaded for scan: %d", r.hmapInputProvider.Count())
+	}
 }
 
 func (r *Runner) readNewTemplatesWithVersionFile(version string) ([]string, error) {

v2/internal/runner/update.go

@@ -86,7 +86,7 @@ func (r *Runner) updateTemplates() error { // TODO this method does more than ju
 	}
 
 	// if disable update check flag is passed and no update template flag is passed
-	if r.options.NoUpdateTemplates && !r.options.UpdateTemplates {
+	if (r.options.NoUpdateTemplates && !r.options.UpdateTemplates) || r.options.Cloud {
 		return nil
 	}
 

v2/pkg/catalog/catalogue.go

@@ -12,7 +12,7 @@ type Catalog interface {
 	// or folders provided as in.
 	GetTemplatePath(target string) ([]string, error)
 	// GetTemplatesPath returns a list of absolute paths for the provided template list.
-	GetTemplatesPath(definitions []string) []string
+	GetTemplatesPath(definitions []string) ([]string, map[string]error)
 	// ResolvePath resolves the path to an absolute one in various ways.
 	//
 	// It checks if the filename is an absolute path, looks in the current directory

v2/pkg/catalog/disk/find.go

@@ -7,14 +7,14 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
-	"github.com/projectdiscovery/gologger"
 )
 
 // GetTemplatesPath returns a list of absolute paths for the provided template list.
-func (c *DiskCatalog) GetTemplatesPath(definitions []string) []string {
+func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[string]error) {
 	// keeps track of processed dirs and files
 	processed := make(map[string]bool)
 	allTemplates := []string{}
+	erred := make(map[string]error)
 
 	for _, t := range definitions {
 		if strings.HasPrefix(t, "http") && (strings.HasSuffix(t, ".yaml") || strings.HasSuffix(t, ".yml")) {
@@ -25,7 +25,7 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) []string {
 		} else {
 			paths, err := c.GetTemplatePath(t)
 			if err != nil {
-				gologger.Error().Msgf("Could not find template '%s': %s\n", t, err)
+				erred[t] = err
 			}
 			for _, path := range paths {
 				if _, ok := processed[path]; !ok {
@@ -35,7 +35,7 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) []string {
 			}
 		}
 	}
-	return allTemplates
+	return allTemplates, erred
 }
 
 // GetTemplatePath parses the specified input template path and returns a compiled

v2/pkg/catalog/disk/path.go

@@ -6,8 +6,6 @@ import (
 	"path/filepath"
 
 	"github.com/pkg/errors"
-
-	folderutil "github.com/projectdiscovery/utils/folder"
 )
 
 // ResolvePath resolves the path to an absolute one in various ways.
@@ -49,20 +47,8 @@ var errNoValidCombination = errors.New("no valid combination found")
 
 // tryResolve attempts to load locate the target by iterating across all the folders tree
 func (c *DiskCatalog) tryResolve(fullPath string) (string, error) {
-	dir, filename := filepath.Split(fullPath)
+	if _, err := os.Stat(fullPath); !os.IsNotExist(err) {
-	pathInfo, err := folderutil.NewPathInfo(dir)
+		return fullPath, nil
-	if err != nil {
-		return "", err
 	}
-	pathInfoItems, err := pathInfo.MeshWith(filename)
-	if err != nil {
-		return "", err
-	}
-	for _, pathInfoItem := range pathInfoItems {
-		if _, err := os.Stat(pathInfoItem); !os.IsNotExist(err) {
-			return pathInfoItem, nil
-		}
-	}
-
 	return "", errNoValidCombination
 }

v2/pkg/catalog/loader/filter/path_filter.go

@@ -18,12 +18,13 @@ type PathFilterConfig struct {
 
 // NewPathFilter creates a new path filter from provided config
 func NewPathFilter(config *PathFilterConfig, catalogClient catalog.Catalog) *PathFilter {
+	paths, _ := catalogClient.GetTemplatesPath(config.ExcludedTemplates)
 	filter := &PathFilter{
-		excludedTemplates:          catalogClient.GetTemplatesPath(config.ExcludedTemplates),
+		excludedTemplates:          paths,
 		alwaysIncludedTemplatesMap: make(map[string]struct{}),
 	}
 
-	alwaysIncludeTemplates := catalogClient.GetTemplatesPath(config.IncludedTemplates)
+	alwaysIncludeTemplates, _ := catalogClient.GetTemplatesPath(config.IncludedTemplates)
 	for _, tpl := range alwaysIncludeTemplates {
 		filter.alwaysIncludedTemplatesMap[tpl] = struct{}{}
 	}

v2/pkg/catalog/loader/loader.go

@@ -58,6 +58,10 @@ type Store struct {
 	workflows []*templates.Template
 
 	preprocessor templates.Preprocessor
+
+	// NotFoundCallback is called for each not found template
+	// This overrides error handling for not found templatesss
+	NotFoundCallback func(template string) bool
 }
 
 // NewConfig returns a new loader config
@@ -174,8 +178,10 @@ func init() {
 // ValidateTemplates takes a list of templates and validates them
 // erroring out on discovering any faulty templates.
 func (store *Store) ValidateTemplates() error {
-	templatePaths := store.config.Catalog.GetTemplatesPath(store.finalTemplates)
+	templatePaths, errs := store.config.Catalog.GetTemplatesPath(store.finalTemplates)
-	workflowPaths := store.config.Catalog.GetTemplatesPath(store.finalWorkflows)
+	store.logErroredTemplates(errs)
+	workflowPaths, errs := store.config.Catalog.GetTemplatesPath(store.finalWorkflows)
+	store.logErroredTemplates(errs)
 
 	filteredTemplatePaths := store.pathFilter.Match(templatePaths)
 	filteredWorkflowPaths := store.pathFilter.Match(workflowPaths)
@@ -263,7 +269,8 @@ func isParsingError(message string, template string, err error) bool {
 
 // LoadTemplates takes a list of templates and returns paths for them
 func (store *Store) LoadTemplates(templatesList []string) []*templates.Template {
-	includedTemplates := store.config.Catalog.GetTemplatesPath(templatesList)
+	includedTemplates, errs := store.config.Catalog.GetTemplatesPath(templatesList)
+	store.logErroredTemplates(errs)
 	templatePathMap := store.pathFilter.Match(includedTemplates)
 
 	loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
@@ -295,7 +302,8 @@ func (store *Store) LoadTemplates(templatesList []string) []*templates.Template
 
 // LoadWorkflows takes a list of workflows and returns paths for them
 func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
-	includedWorkflows := store.config.Catalog.GetTemplatesPath(workflowsList)
+	includedWorkflows, errs := store.config.Catalog.GetTemplatesPath(workflowsList)
+	store.logErroredTemplates(errs)
 	workflowPathMap := store.pathFilter.Match(includedWorkflows)
 
 	loadedWorkflows := make([]*templates.Template, 0, len(workflowPathMap))
@@ -319,7 +327,8 @@ func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template
 // LoadTemplatesWithTags takes a list of templates and extra tags
 // returning templates that match.
 func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templates.Template {
-	includedTemplates := store.config.Catalog.GetTemplatesPath(templatesList)
+	includedTemplates, errs := store.config.Catalog.GetTemplatesPath(templatesList)
+	store.logErroredTemplates(errs)
 	templatePathMap := store.pathFilter.Match(includedTemplates)
 
 	loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
@@ -382,3 +391,11 @@ func workflowContainsProtocol(workflow []*workflows.WorkflowTemplate) bool {
 	}
 	return false
 }
+
+func (s *Store) logErroredTemplates(erred map[string]error) {
+	for template, err := range erred {
+		if s.NotFoundCallback == nil || !s.NotFoundCallback(template) {
+			gologger.Error().Msgf("Could not find template '%s': %s", template, err)
+		}
+	}
+}

v2/pkg/core/inputs/hybrid/hmap.go

@@ -41,9 +41,20 @@ type Input struct {
 	sync.Once
 }
 
+// Options is a wrapper around types.Options structure
+type Options struct {
+	// Options contains options for hmap provider
+	Options *types.Options
+	// NotFoundCallback is called for each not found target
+	// This overrides error handling for not found target
+	NotFoundCallback func(template string) bool
+}
+
 // New creates a new hmap backed nuclei Input Provider
 // and initializes it based on the passed options Model.
-func New(options *types.Options) (*Input, error) {
+func New(opts *Options) (*Input, error) {
+	options := opts.Options
+
 	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create temporary input file")
@@ -71,7 +82,7 @@ func New(options *types.Options) (*Input, error) {
 		}
 		input.hostMapStream = fkv
 	}
-	if initErr := input.initializeInputSources(options); initErr != nil {
+	if initErr := input.initializeInputSources(opts); initErr != nil {
 		return nil, initErr
 	}
 	if input.dupeCount > 0 {
@@ -89,7 +100,9 @@ func (i *Input) Close() {
 }
 
 // initializeInputSources initializes the input sources for hmap input
-func (i *Input) initializeInputSources(options *types.Options) error {
+func (i *Input) initializeInputSources(opts *Options) error {
+	options := opts.Options
+
 	// Handle targets flags
 	for _, target := range options.Targets {
 		switch {
@@ -111,11 +124,15 @@ func (i *Input) initializeInputSources(options *types.Options) error {
 	if options.TargetsFilePath != "" {
 		input, inputErr := os.Open(options.TargetsFilePath)
 		if inputErr != nil {
-			return errors.Wrap(inputErr, "could not open targets file")
+			// Handle cloud based input here.
+			if opts.NotFoundCallback == nil || !opts.NotFoundCallback(options.TargetsFilePath) {
+				return errors.Wrap(inputErr, "could not open targets file")
+			}
+		}
+		if input != nil {
+			i.scanInputFromReader(input)
+			input.Close()
 		}
-		defer input.Close()
-
-		i.scanInputFromReader(input)
 	}
 	if options.Uncover && options.UncoverQuery != nil {
 		gologger.Info().Msgf("Running uncover query against: %s", strings.Join(options.UncoverEngine, ","))

v2/pkg/external/customtemplates/github.go

@@ -30,7 +30,7 @@ func (customTemplate *customTemplateGithubRepo) Download(location string, ctx co
 	if !fileutil.FolderExists(clonePath) {
 		err := customTemplate.cloneRepo(clonePath, customTemplate.githubToken)
 		if err != nil {
-			gologger.Info().Msgf("%s", err)
+			gologger.Error().Msgf("%s", err)
 		} else {
 			gologger.Info().Msgf("Repo %s/%s cloned successfully at %s", customTemplate.owner, customTemplate.reponame, clonePath)
 		}
@@ -49,7 +49,7 @@ func (customTemplate *customTemplateGithubRepo) Update(location string, ctx cont
 	}
 	err := customTemplate.pullChanges(clonePath, customTemplate.githubToken)
 	if err != nil {
-		gologger.Info().Msgf("%s", err)
+		gologger.Error().Msgf("%s", err)
 	} else {
 		gologger.Info().Msgf("Repo %s/%s successfully pulled the changes.\n", customTemplate.owner, customTemplate.reponame)
 	}

v2/pkg/external/customtemplates/templates_provider.go

@@ -21,18 +21,21 @@ type Provider interface {
 // parseCustomTemplates function reads the options.GithubTemplateRepo list,
 // Checks the given repos are valid or not and stores them into runner.CustomTemplates
 func ParseCustomTemplates(options *types.Options) []Provider {
+	if options.Cloud {
+		return nil
+	}
 	var customTemplates []Provider
 	gitHubClient := getGHClientIncognito()
 
 	for _, repoName := range options.GithubTemplateRepo {
 		owner, repo, err := getOwnerAndRepo(repoName)
 		if err != nil {
-			gologger.Info().Msgf("%s", err)
+			gologger.Error().Msgf("%s", err)
 			continue
 		}
 		githubRepo, err := getGithubRepo(gitHubClient, owner, repo, options.GithubToken)
 		if err != nil {
-			gologger.Info().Msgf("%s", err)
+			gologger.Error().Msgf("%s", err)
 			continue
 		}
 		customTemplateRepo := &customTemplateGithubRepo{

v2/pkg/model/types/severity/severities.go

@@ -68,6 +68,14 @@ func (severities Severities) String() string {
 	return strings.Join(stringSeverities, ", ")
 }
 
+func (severities Severities) MarshalJSON() ([]byte, error) {
+	var stringSeverities = make([]string, 0, len(severities))
+	for _, severity := range severities {
+		stringSeverities = append(stringSeverities, severity.String())
+	}
+	return json.Marshal(stringSeverities)
+}
+
 func setSeverity(severities *Severities, value string) error {
 	computedSeverity, err := toSeverity(value)
 	if err != nil {

v2/pkg/model/types/severity/severity_test.go

@@ -6,6 +6,7 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestYamlUnmarshal(t *testing.T) {
@@ -65,3 +66,12 @@ func unmarshal(value string, unmarshaller func(data []byte, v interface{}) error
 func createYAML(value string) string {
 	return "severity: " + value + "\n"
 }
+
+func TestMarshalJSON(t *testing.T) {
+	unmarshalled := Severities{Low, Medium}
+	data, err := unmarshalled.MarshalJSON()
+	if err != nil {
+		panic(err)
+	}
+	require.Equal(t, "[\"low\",\"medium\"]", string(data), "could not marshal json")
+}

v2/pkg/parsers/workflow_loader.go

@@ -40,7 +40,10 @@ func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error)
 }
 
 func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string {
-	includedTemplates := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory})
+	includedTemplates, errs := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory})
+	for template, err := range errs {
+		gologger.Error().Msgf("Could not find template '%s': %s", template, err)
+	}
 	templatePathMap := w.pathFilter.Match(includedTemplates)
 
 	loadedTemplates := make([]string, 0, len(templatePathMap))
@@ -56,7 +59,10 @@ func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string
 }
 
 func (w *workflowLoader) GetTemplatePaths(templatesList []string, noValidate bool) []string {
-	includedTemplates := w.options.Catalog.GetTemplatesPath(templatesList)
+	includedTemplates, errs := w.options.Catalog.GetTemplatesPath(templatesList)
+	for template, err := range errs {
+		gologger.Error().Msgf("Could not find template '%s': %s", template, err)
+	}
 	templatesPathMap := w.pathFilter.Match(includedTemplates)
 
 	loadedTemplates := make([]string, 0, len(templatesPathMap))

v2/pkg/templates/types/types.go

@@ -154,6 +154,14 @@ func (protocolTypes *ProtocolTypes) UnmarshalYAML(unmarshal func(interface{}) er
 	return nil
 }
 
+func (protocolTypes ProtocolTypes) MarshalJSON() ([]byte, error) {
+	var stringProtocols = make([]string, 0, len(protocolTypes))
+	for _, protocol := range protocolTypes {
+		stringProtocols = append(stringProtocols, protocol.String())
+	}
+	return json.Marshal(stringProtocols)
+}
+
 func (protocolTypes ProtocolTypes) String() string {
 	var stringTypes []string
 	for _, t := range protocolTypes {

v2/pkg/types/types.go

@@ -97,10 +97,34 @@ type Options struct {
 	CloudAPIKey string
 	// Scanlist feature to get all the scan ids for a user
 	ScanList bool
+	// ListDatasources enables listing of datasources for user
+	ListDatasources bool
+	// ListTargets enables listing of targets for user
+	ListTargets bool
+	// ListTemplates enables listing of templates for user
+	ListTemplates bool
+	// Limit the number of items at a time
+	OutputLimit int
 	// Nostore
 	NoStore bool
 	// Delete scan
 	DeleteScan string
+	// AddDatasource adds a datasource to cloud storage
+	AddDatasource string
+	// RemoveDatasource deletes a datasource from cloud storage
+	RemoveDatasource string
+	// AddTemplate adds a list of templates to custom datasource
+	AddTemplate string
+	// AddTarget adds a list of targets to custom datasource
+	AddTarget string
+	// GetTemplate gets a template by id
+	GetTemplate string
+	// GetTarget gets a target by id
+	GetTarget string
+	// RemoveTemplate removes a list of templates
+	RemoveTemplate string
+	// RemoveTarget removes a list of targets
+	RemoveTarget string
 	// Get issues for a scan
 	ScanOutput string
 	// ResolversFile is a file containing resolvers for nuclei.
@@ -157,6 +181,8 @@ type Options struct {
 	Headless bool
 	// ShowBrowser specifies whether the show the browser in headless mode
 	ShowBrowser bool
+	// NoTables disables pretty printing of cloud results in tables
+	NoTables bool
 	// DisableClustering disables clustering of templates
 	DisableClustering bool
 	// UseInstalledChrome skips chrome install and use local instance
@@ -352,3 +378,8 @@ func DefaultOptions() *Options {
 		MaxHostError:            30,
 	}
 }
+
+// HasCloudOptions returns true if cloud options have been specified
+func (options *Options) HasCloudOptions() bool {
+	return options.ScanList || options.DeleteScan != "" || options.ScanOutput != "" || options.ListDatasources || options.ListTargets || options.ListTemplates || options.RemoveDatasource != "" || options.AddTarget != "" || options.AddTemplate != "" || options.RemoveTarget != "" || options.RemoveTemplate != "" || options.GetTarget != "" || options.GetTemplate != ""
+}