Feature 18 cloud flags (#2708)

* Add cloud flags for nuclei.

* Add flag to get output for a particular scan ID

* Add some comments to the function.

* Get timestamp and id for scan list

* Fix linting errors

* Check if type is enumeration.

* Do not show deleted scans.

* Do not use filter_result, create client once and use it everywhere with
runner.

* Fix the output of scan list to be better

* Format the nuclei scan output list.

* Remove unused constant

* misc option update

Co-authored-by: sandeep <sandeep@projectdiscovery.io>

v2/cmd/nuclei/main.go

@@ -280,6 +280,10 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVar(&options.Cloud, "cloud", false, "run scan on nuclei cloud"),
 		flagSet.StringVarEnv(&options.CloudURL, "cloud-server", "cs", "http://cloud-dev.nuclei.sh", "NUCLEI_CLOUD_SERVER", "nuclei cloud server to use"),
 		flagSet.StringVarEnv(&options.CloudAPIKey, "cloud-api-key", "ak", "", "NUCLEI_CLOUD_APIKEY", "api-key for the nuclei cloud server"),
+		flagSet.BoolVarP(&options.ScanList, "list-scan", "ls", false, "list cloud scans."),
+		flagSet.BoolVarP(&options.NoStore, "no-store", "ns", false, "disable scan/output storage on cloud"),
+		flagSet.StringVarP(&options.DeleteScan, "delete-scan", "ds", "", "delete scan/output on cloud by scan id"),
+		flagSet.StringVarP(&options.ScanOutput, "scan-output", "so", "", "display scan output by scan id"),
 	)
 
 	_ = flagSet.Parse()

v2/internal/runner/enumerate.go

@@ -14,6 +14,8 @@ import (
 	"go.uber.org/atomic"
 )
 
+const DDMMYYYYhhmmss = "2006-01-02 15:04:05"
+
 // runStandardEnumeration runs standard enumeration
 func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions, store *loader.Store, engine *core.Engine) (*atomic.Bool, error) {
 	if r.options.AutomaticScan {
@@ -22,13 +24,53 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions,
 	return r.executeTemplatesInput(store, engine)
 }
 
+// Get all the scan lists for a user/apikey.
+func (r *Runner) getScanList() error {
+	items, err := r.cloudClient.GetScans()
+	loc, _ := time.LoadLocation("Local")
+
+	for _, v := range items {
+		status := "FINISHED"
+		t := v.FinishedAt
+		duration := t.Sub(v.CreatedAt)
+		if !v.Finished {
+			status = "RUNNING"
+			t = time.Now().UTC()
+			duration = t.Sub(v.CreatedAt)
+		}
+
+		val := v.CreatedAt.In(loc).Format(DDMMYYYYhhmmss)
+
+		gologger.Silent().Msgf("%s [%s] [STATUS: %s] [MATCHED: %d] [TARGETS: %d] [TEMPLATES: %d] [DURATION: %s]\n", v.Id, val, status, v.Matches, v.Targets, v.Templates, duration)
+	}
+	return err
+}
+
+func (r *Runner) deleteScan(id string) error {
+	deleted, err := r.cloudClient.DeleteScan(id)
+	if !deleted.OK {
+		gologger.Info().Msgf("Error in deleting the scan %s.", id)
+	} else {
+		gologger.Info().Msgf("Scan deleted %s.", id)
+	}
+	return err
+}
+
+func (r *Runner) getResults(id string) error {
+	err := r.cloudClient.GetResults(id, func(re *output.ResultEvent) {
+		if outputErr := r.output.Write(re); outputErr != nil {
+			gologger.Warning().Msgf("Could not write output: %s", outputErr)
+		}
+	}, false)
+	return err
+}
+
 // runCloudEnumeration runs cloud based enumeration
-func (r *Runner) runCloudEnumeration(store *loader.Store) (*atomic.Bool, error) {
+func (r *Runner) runCloudEnumeration(store *loader.Store, nostore bool) (*atomic.Bool, error) {
 	now := time.Now()
 	defer func() {
 		gologger.Info().Msgf("Scan execution took %s", time.Since(now))
 	}()
-	client := nucleicloud.New(r.options.CloudURL, r.options.CloudAPIKey)
 
 	results := &atomic.Bool{}
 
@@ -41,9 +83,10 @@ func (r *Runner) runCloudEnumeration(store *loader.Store) (*atomic.Bool, error)
 	for _, template := range store.Templates() {
 		templates = append(templates, getTemplateRelativePath(template.Path))
 	}
-	taskID, err := client.AddScan(&nucleicloud.AddScanRequest{
+	taskID, err := r.cloudClient.AddScan(&nucleicloud.AddScanRequest{
 		RawTargets:      targets,
 		PublicTemplates: templates,
+		IsTemporary:     nostore,
 	})
 	if err != nil {
 		return results, err
@@ -51,7 +94,7 @@ func (r *Runner) runCloudEnumeration(store *loader.Store) (*atomic.Bool, error)
 	gologger.Info().Msgf("Created task with ID: %s", taskID)
 	time.Sleep(3 * time.Second)
 
-	err = client.GetResults(taskID, func(re *output.ResultEvent) {
+	err = r.cloudClient.GetResults(taskID, func(re *output.ResultEvent) {
 		results.CompareAndSwap(false, true)
 
 		if outputErr := r.output.Write(re); outputErr != nil {
@@ -62,7 +105,7 @@ func (r *Runner) runCloudEnumeration(store *loader.Store) (*atomic.Bool, error)
 				gologger.Warning().Msgf("Could not create issue on tracker: %s", err)
 			}
 		}
-	})
+	}, true)
 	return results, err
 }
 

v2/internal/runner/nucleicloud/cloud.go

@@ -24,6 +24,7 @@ type Client struct {
 const (
 	pollInterval   = 1 * time.Second
 	defaultBaseURL = "http://webapp.localhost"
+	resultSize     = 100
 )
 
 // New returns a nuclei-cloud API client
@@ -72,10 +73,11 @@ func (c *Client) AddScan(req *AddScanRequest) (string, error) {
 
 // GetResults gets results from nuclei server for an ID
 // until there are no more results left to retrieve.
-func (c *Client) GetResults(ID string, callback func(*output.ResultEvent)) error {
+func (c *Client) GetResults(ID string, callback func(*output.ResultEvent), checkProgress bool) error {
 	lastID := int64(0)
 	for {
-		httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/results?id=%s&from=%d&size=100", c.baseURL, ID, lastID), nil)
+		uri := fmt.Sprintf("%s/results?id=%s&from=%d&size=%d", c.baseURL, ID, lastID, resultSize)
+		httpReq, err := retryablehttp.NewRequest(http.MethodGet, uri, nil)
 		if err != nil {
 			return errors.Wrap(err, "could not make request")
 		}
@@ -106,10 +108,76 @@ func (c *Client) GetResults(ID string, callback func(*output.ResultEvent)) error
 			}
 			callback(&result)
 		}
-		if items.Finished && len(items.Items) == 0 {
+
+		//This is checked during scan is added else if no item found break out of loop.
+		if checkProgress {
+			if items.Finished && len(items.Items) == 0 {
+				break
+			}
+		} else if len(items.Items) == 0 {
 			break
 		}
+
 		time.Sleep(pollInterval)
 	}
 	return nil
 }
+
+func (c *Client) GetScans() ([]GetScanRequest, error) {
+	var items []GetScanRequest
+	httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/scan", c.baseURL), nil)
+	if err != nil {
+		return items, errors.Wrap(err, "could not make request")
+	}
+	httpReq.Header.Set("X-API-Key", c.apiKey)
+
+	resp, err := c.httpclient.Do(httpReq)
+	if err != nil {
+		return items, errors.Wrap(err, "could not make request.")
+	}
+	if err != nil {
+		return items, errors.Wrap(err, "could not do get response.")
+	}
+	if resp.StatusCode != 200 {
+		data, _ := io.ReadAll(resp.Body)
+		resp.Body.Close()
+		return items, errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data))
+	}
+	if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil {
+		resp.Body.Close()
+		return items, errors.Wrap(err, "could not decode results")
+	}
+	resp.Body.Close()
+
+	return items, nil
+}
+
+//Delete a scan and it's issues by the scan id.
+func (c *Client) DeleteScan(id string) (DeleteScanResults, error) {
+	deletescan := DeleteScanResults{}
+	httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/scan?id=%s", c.baseURL, id), nil)
+	if err != nil {
+		return deletescan, errors.Wrap(err, "could not make request")
+	}
+	httpReq.Header.Set("X-API-Key", c.apiKey)
+
+	resp, err := c.httpclient.Do(httpReq)
+	if err != nil {
+		return deletescan, errors.Wrap(err, "could not make request")
+	}
+	if err != nil {
+		return deletescan, errors.Wrap(err, "could not do get result request")
+	}
+	if resp.StatusCode != 200 {
+		data, _ := io.ReadAll(resp.Body)
+		resp.Body.Close()
+		return deletescan, errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data))
+	}
+	if err := jsoniter.NewDecoder(resp.Body).Decode(&deletescan); err != nil {
+		resp.Body.Close()
+		return deletescan, errors.Wrap(err, "could not delete scan")
+	}
+	resp.Body.Close()
+
+	return deletescan, nil
+}

v2/internal/runner/nucleicloud/types.go

@@ -1,5 +1,7 @@
 package nucleicloud
 
+import "time"
+
 // AddScanRequest is a nuclei scan input item.
 type AddScanRequest struct {
 	// RawTargets is a list of raw target URLs for the scan.
@@ -9,6 +11,7 @@ type AddScanRequest struct {
 	// PrivateTemplates is a map of template-name->contents that
 	// are private to the user executing the scan. (TODO: TBD)
 	PrivateTemplates map[string]string `json:"private_templates,omitempty"`
+	IsTemporary      bool              `json:"is_temporary"`
 }
 
 type GetResultsResponse struct {
@@ -16,7 +19,23 @@ type GetResultsResponse struct {
 	Items    []GetResultsResponseItem `json:"items"`
 }
 
+type GetScanRequest struct {
+	Id         string    `json:"id"`
+	Total      int32     `json:"total"`
+	Current    int32     `json:"current"`
+	Finished   bool      `json:"finished"`
+	CreatedAt  time.Time `json:"created_at"`
+	FinishedAt time.Time `json:"finished_at"`
+	Targets    int32     `json:"targets"`
+	Templates  int32     `json:"templates"`
+	Matches    int64     `json:"matches"`
+}
+
 type GetResultsResponseItem struct {
 	ID  int64  `json:"id"`
 	Raw string `json:"raw"`
 }
+
+type DeleteScanResults struct {
+	OK bool `json:"ok"`
+}

v2/internal/runner/runner.go

@@ -14,6 +14,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/projectdiscovery/nuclei/v2/internal/runner/nucleicloud"
+
 	"github.com/blang/semver"
 	"github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
@@ -70,6 +72,7 @@ type Runner struct {
 	hostErrors        hosterrorscache.CacheInterface
 	resumeCfg         *types.ResumeCfg
 	pprofServer       *http.Server
+	cloudClient       *nucleicloud.Client
 }
 
 const pprofServerAddress = "127.0.0.1:8086"
@@ -85,6 +88,10 @@ func New(options *types.Options) (*Runner, error) {
 		os.Exit(0)
 	}
 
+	if options.Cloud {
+		runner.cloudClient = nucleicloud.New(options.CloudURL, options.CloudAPIKey)
+	}
+
 	if options.UpdateNuclei {
 		if err := updateNucleiVersionToLatest(runner.options.Verbose); err != nil {
 			return nil, err
@@ -418,12 +425,27 @@ func (r *Runner) RunEnumeration() error {
 		executerOpts.InputHelper.InputsHTTP = inputHelpers
 	}
 
+	enumeration := false
 	var results *atomic.Bool
 	if r.options.Cloud {
-		gologger.Info().Msgf("Running scan on cloud with URL %s", r.options.CloudURL)
+		if r.options.ScanList {
-		results, err = r.runCloudEnumeration(store)
+			err = r.getScanList()
+		} else if r.options.DeleteScan != "" {
+			err = r.deleteScan(r.options.DeleteScan)
+		} else if r.options.ScanOutput != "" {
+			err = r.getResults(r.options.ScanOutput)
+		} else {
+			gologger.Info().Msgf("Running scan on cloud with URL %s", r.options.CloudURL)
+			results, err = r.runCloudEnumeration(store, r.options.NoStore)
+			enumeration = true
+		}
 	} else {
 		results, err = r.runStandardEnumeration(executerOpts, store, engine)
+		enumeration = true
+	}
+
+	if !enumeration {
+		return err
 	}
 
 	if r.interactsh != nil {

v2/pkg/types/types.go

@@ -95,6 +95,14 @@ type Options struct {
 	CloudURL string
 	// CloudAPIKey is the api-key for the nuclei cloud endpoint
 	CloudAPIKey string
+	// Scanlist feature to get all the scan ids for a user
+	ScanList bool
+	// Nostore
+	NoStore bool
+	// Delete scan
+	DeleteScan string
+	// Get issues for a scan
+	ScanOutput string
 	// ResolversFile is a file containing resolvers for nuclei.
 	ResolversFile string
 	// StatsInterval is the number of seconds to display stats after