From 4cfde111f47b54a317e481f2a658388524d2dd9b Mon Sep 17 00:00:00 2001 From: Jaideep Khandelwal Date: Sat, 22 Oct 2022 04:06:52 +0530 Subject: [PATCH] Feature 18 cloud flags (#2708) * Add cloud flags for nuclei. * Add flag to get output for a particular scan ID * Add some comments to the function. * Get timestamp and id for scan list * Fix linting errors * Check if type is enumeration. * Do not show deleted scans. * Do not use filter_result, create client once and use it everywhere with runner. * Fix the output of scan list to be better * Format the nuclei scan output list. * Remove unused constant * misc option update Co-authored-by: sandeep --- v2/cmd/nuclei/main.go | 4 ++ v2/internal/runner/enumerate.go | 53 ++++++++++++++++-- v2/internal/runner/nucleicloud/cloud.go | 74 ++++++++++++++++++++++++- v2/internal/runner/nucleicloud/types.go | 19 +++++++ v2/internal/runner/runner.go | 26 ++++++++- v2/pkg/types/types.go | 8 +++ 6 files changed, 174 insertions(+), 10 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 397bb749..0f05d91c 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -280,6 +280,10 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.Cloud, "cloud", false, "run scan on nuclei cloud"), flagSet.StringVarEnv(&options.CloudURL, "cloud-server", "cs", "http://cloud-dev.nuclei.sh", "NUCLEI_CLOUD_SERVER", "nuclei cloud server to use"), flagSet.StringVarEnv(&options.CloudAPIKey, "cloud-api-key", "ak", "", "NUCLEI_CLOUD_APIKEY", "api-key for the nuclei cloud server"), + flagSet.BoolVarP(&options.ScanList, "list-scan", "ls", false, "list cloud scans."), + flagSet.BoolVarP(&options.NoStore, "no-store", "ns", false, "disable scan/output storage on cloud"), + flagSet.StringVarP(&options.DeleteScan, "delete-scan", "ds", "", "delete scan/output on cloud by scan id"), + flagSet.StringVarP(&options.ScanOutput, "scan-output", "so", "", "display scan output by scan id"), ) _ = flagSet.Parse() diff --git a/v2/internal/runner/enumerate.go b/v2/internal/runner/enumerate.go index 3e49182d..eddd7653 100644 --- a/v2/internal/runner/enumerate.go +++ b/v2/internal/runner/enumerate.go @@ -14,6 +14,8 @@ import ( "go.uber.org/atomic" ) +const DDMMYYYYhhmmss = "2006-01-02 15:04:05" + // runStandardEnumeration runs standard enumeration func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions, store *loader.Store, engine *core.Engine) (*atomic.Bool, error) { if r.options.AutomaticScan { @@ -22,13 +24,53 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions, return r.executeTemplatesInput(store, engine) } +// Get all the scan lists for a user/apikey. +func (r *Runner) getScanList() error { + items, err := r.cloudClient.GetScans() + loc, _ := time.LoadLocation("Local") + + for _, v := range items { + status := "FINISHED" + t := v.FinishedAt + duration := t.Sub(v.CreatedAt) + if !v.Finished { + status = "RUNNING" + t = time.Now().UTC() + duration = t.Sub(v.CreatedAt) + } + + val := v.CreatedAt.In(loc).Format(DDMMYYYYhhmmss) + + gologger.Silent().Msgf("%s [%s] [STATUS: %s] [MATCHED: %d] [TARGETS: %d] [TEMPLATES: %d] [DURATION: %s]\n", v.Id, val, status, v.Matches, v.Targets, v.Templates, duration) + } + return err +} + +func (r *Runner) deleteScan(id string) error { + deleted, err := r.cloudClient.DeleteScan(id) + if !deleted.OK { + gologger.Info().Msgf("Error in deleting the scan %s.", id) + } else { + gologger.Info().Msgf("Scan deleted %s.", id) + } + return err +} + +func (r *Runner) getResults(id string) error { + err := r.cloudClient.GetResults(id, func(re *output.ResultEvent) { + if outputErr := r.output.Write(re); outputErr != nil { + gologger.Warning().Msgf("Could not write output: %s", outputErr) + } + }, false) + return err +} + // runCloudEnumeration runs cloud based enumeration -func (r *Runner) runCloudEnumeration(store *loader.Store) (*atomic.Bool, error) { +func (r *Runner) runCloudEnumeration(store *loader.Store, nostore bool) (*atomic.Bool, error) { now := time.Now() defer func() { gologger.Info().Msgf("Scan execution took %s", time.Since(now)) }() - client := nucleicloud.New(r.options.CloudURL, r.options.CloudAPIKey) results := &atomic.Bool{} @@ -41,9 +83,10 @@ func (r *Runner) runCloudEnumeration(store *loader.Store) (*atomic.Bool, error) for _, template := range store.Templates() { templates = append(templates, getTemplateRelativePath(template.Path)) } - taskID, err := client.AddScan(&nucleicloud.AddScanRequest{ + taskID, err := r.cloudClient.AddScan(&nucleicloud.AddScanRequest{ RawTargets: targets, PublicTemplates: templates, + IsTemporary: nostore, }) if err != nil { return results, err @@ -51,7 +94,7 @@ func (r *Runner) runCloudEnumeration(store *loader.Store) (*atomic.Bool, error) gologger.Info().Msgf("Created task with ID: %s", taskID) time.Sleep(3 * time.Second) - err = client.GetResults(taskID, func(re *output.ResultEvent) { + err = r.cloudClient.GetResults(taskID, func(re *output.ResultEvent) { results.CompareAndSwap(false, true) if outputErr := r.output.Write(re); outputErr != nil { @@ -62,7 +105,7 @@ func (r *Runner) runCloudEnumeration(store *loader.Store) (*atomic.Bool, error) gologger.Warning().Msgf("Could not create issue on tracker: %s", err) } } - }) + }, true) return results, err } diff --git a/v2/internal/runner/nucleicloud/cloud.go b/v2/internal/runner/nucleicloud/cloud.go index 77bb6adc..c45bf000 100644 --- a/v2/internal/runner/nucleicloud/cloud.go +++ b/v2/internal/runner/nucleicloud/cloud.go @@ -24,6 +24,7 @@ type Client struct { const ( pollInterval = 1 * time.Second defaultBaseURL = "http://webapp.localhost" + resultSize = 100 ) // New returns a nuclei-cloud API client @@ -72,10 +73,11 @@ func (c *Client) AddScan(req *AddScanRequest) (string, error) { // GetResults gets results from nuclei server for an ID // until there are no more results left to retrieve. -func (c *Client) GetResults(ID string, callback func(*output.ResultEvent)) error { +func (c *Client) GetResults(ID string, callback func(*output.ResultEvent), checkProgress bool) error { lastID := int64(0) for { - httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/results?id=%s&from=%d&size=100", c.baseURL, ID, lastID), nil) + uri := fmt.Sprintf("%s/results?id=%s&from=%d&size=%d", c.baseURL, ID, lastID, resultSize) + httpReq, err := retryablehttp.NewRequest(http.MethodGet, uri, nil) if err != nil { return errors.Wrap(err, "could not make request") } @@ -106,10 +108,76 @@ func (c *Client) GetResults(ID string, callback func(*output.ResultEvent)) error } callback(&result) } - if items.Finished && len(items.Items) == 0 { + + //This is checked during scan is added else if no item found break out of loop. + if checkProgress { + if items.Finished && len(items.Items) == 0 { + break + } + } else if len(items.Items) == 0 { break } + time.Sleep(pollInterval) } return nil } + +func (c *Client) GetScans() ([]GetScanRequest, error) { + var items []GetScanRequest + httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/scan", c.baseURL), nil) + if err != nil { + return items, errors.Wrap(err, "could not make request") + } + httpReq.Header.Set("X-API-Key", c.apiKey) + + resp, err := c.httpclient.Do(httpReq) + if err != nil { + return items, errors.Wrap(err, "could not make request.") + } + if err != nil { + return items, errors.Wrap(err, "could not do get response.") + } + if resp.StatusCode != 200 { + data, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return items, errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data)) + } + if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil { + resp.Body.Close() + return items, errors.Wrap(err, "could not decode results") + } + resp.Body.Close() + + return items, nil +} + +//Delete a scan and it's issues by the scan id. +func (c *Client) DeleteScan(id string) (DeleteScanResults, error) { + deletescan := DeleteScanResults{} + httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/scan?id=%s", c.baseURL, id), nil) + if err != nil { + return deletescan, errors.Wrap(err, "could not make request") + } + httpReq.Header.Set("X-API-Key", c.apiKey) + + resp, err := c.httpclient.Do(httpReq) + if err != nil { + return deletescan, errors.Wrap(err, "could not make request") + } + if err != nil { + return deletescan, errors.Wrap(err, "could not do get result request") + } + if resp.StatusCode != 200 { + data, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return deletescan, errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data)) + } + if err := jsoniter.NewDecoder(resp.Body).Decode(&deletescan); err != nil { + resp.Body.Close() + return deletescan, errors.Wrap(err, "could not delete scan") + } + resp.Body.Close() + + return deletescan, nil +} diff --git a/v2/internal/runner/nucleicloud/types.go b/v2/internal/runner/nucleicloud/types.go index 14a180a6..c746fe0b 100644 --- a/v2/internal/runner/nucleicloud/types.go +++ b/v2/internal/runner/nucleicloud/types.go @@ -1,5 +1,7 @@ package nucleicloud +import "time" + // AddScanRequest is a nuclei scan input item. type AddScanRequest struct { // RawTargets is a list of raw target URLs for the scan. @@ -9,6 +11,7 @@ type AddScanRequest struct { // PrivateTemplates is a map of template-name->contents that // are private to the user executing the scan. (TODO: TBD) PrivateTemplates map[string]string `json:"private_templates,omitempty"` + IsTemporary bool `json:"is_temporary"` } type GetResultsResponse struct { @@ -16,7 +19,23 @@ type GetResultsResponse struct { Items []GetResultsResponseItem `json:"items"` } +type GetScanRequest struct { + Id string `json:"id"` + Total int32 `json:"total"` + Current int32 `json:"current"` + Finished bool `json:"finished"` + CreatedAt time.Time `json:"created_at"` + FinishedAt time.Time `json:"finished_at"` + Targets int32 `json:"targets"` + Templates int32 `json:"templates"` + Matches int64 `json:"matches"` +} + type GetResultsResponseItem struct { ID int64 `json:"id"` Raw string `json:"raw"` } + +type DeleteScanResults struct { + OK bool `json:"ok"` +} diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 2c07b100..406c47fe 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -14,6 +14,8 @@ import ( "strings" "time" + "github.com/projectdiscovery/nuclei/v2/internal/runner/nucleicloud" + "github.com/blang/semver" "github.com/logrusorgru/aurora" "github.com/pkg/errors" @@ -70,6 +72,7 @@ type Runner struct { hostErrors hosterrorscache.CacheInterface resumeCfg *types.ResumeCfg pprofServer *http.Server + cloudClient *nucleicloud.Client } const pprofServerAddress = "127.0.0.1:8086" @@ -85,6 +88,10 @@ func New(options *types.Options) (*Runner, error) { os.Exit(0) } + if options.Cloud { + runner.cloudClient = nucleicloud.New(options.CloudURL, options.CloudAPIKey) + } + if options.UpdateNuclei { if err := updateNucleiVersionToLatest(runner.options.Verbose); err != nil { return nil, err @@ -418,12 +425,27 @@ func (r *Runner) RunEnumeration() error { executerOpts.InputHelper.InputsHTTP = inputHelpers } + enumeration := false var results *atomic.Bool if r.options.Cloud { - gologger.Info().Msgf("Running scan on cloud with URL %s", r.options.CloudURL) - results, err = r.runCloudEnumeration(store) + if r.options.ScanList { + err = r.getScanList() + } else if r.options.DeleteScan != "" { + err = r.deleteScan(r.options.DeleteScan) + } else if r.options.ScanOutput != "" { + err = r.getResults(r.options.ScanOutput) + } else { + gologger.Info().Msgf("Running scan on cloud with URL %s", r.options.CloudURL) + results, err = r.runCloudEnumeration(store, r.options.NoStore) + enumeration = true + } } else { results, err = r.runStandardEnumeration(executerOpts, store, engine) + enumeration = true + } + + if !enumeration { + return err } if r.interactsh != nil { diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 7d10a9c4..28b6b2b7 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -95,6 +95,14 @@ type Options struct { CloudURL string // CloudAPIKey is the api-key for the nuclei cloud endpoint CloudAPIKey string + // Scanlist feature to get all the scan ids for a user + ScanList bool + // Nostore + NoStore bool + // Delete scan + DeleteScan string + // Get issues for a scan + ScanOutput string // ResolversFile is a file containing resolvers for nuclei. ResolversFile string // StatsInterval is the number of seconds to display stats after