daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Feature 18 cloud flags (#2708) 

* Add cloud flags for nuclei.

* Add flag to get output for a particular scan ID

* Add some comments to the function.

* Get timestamp and id for scan list

* Fix linting errors

* Check if type is enumeration.

* Do not show deleted scans.

* Do not use filter_result, create client once and use it everywhere with
runner.

* Fix the output of scan list to be better

* Format the nuclei scan output list.

* Remove unused constant

* misc option update

Co-authored-by: sandeep <sandeep@projectdiscovery.io>
dev
Jaideep Khandelwal 2022-10-22 04:06:52 +05:30 committed by GitHub
parent 2006060574
commit 4cfde111f4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 174 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
v2/cmd/nuclei/main.go
View File

@ -280,6 +280,10 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.BoolVar(&options.Cloud, "cloud", false, "run scan on nuclei cloud"),
flagSet.StringVarEnv(&options.CloudURL, "cloud-server", "cs", "http://cloud-dev.nuclei.sh", "NUCLEI_CLOUD_SERVER", "nuclei cloud server to use"),
flagSet.StringVarEnv(&options.CloudAPIKey, "cloud-api-key", "ak", "", "NUCLEI_CLOUD_APIKEY", "api-key for the nuclei cloud server"),
flagSet.BoolVarP(&options.ScanList, "list-scan", "ls", false, "list cloud scans."),
flagSet.BoolVarP(&options.NoStore, "no-store", "ns", false, "disable scan/output storage on cloud"),
flagSet.StringVarP(&options.DeleteScan, "delete-scan", "ds", "", "delete scan/output on cloud by scan id"),
flagSet.StringVarP(&options.ScanOutput, "scan-output", "so", "", "display scan output by scan id"),
)
_ = flagSet.Parse()

53
v2/internal/runner/enumerate.go
View File

@ -14,6 +14,8 @@ import (
"go.uber.org/atomic"
)
const DDMMYYYYhhmmss = "2006-01-02 15:04:05"
// runStandardEnumeration runs standard enumeration
func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions, store *loader.Store, engine *core.Engine) (*atomic.Bool, error) {
if r.options.AutomaticScan {
@ -22,13 +24,53 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions,
return r.executeTemplatesInput(store, engine)
}
// Get all the scan lists for a user/apikey.
func (r *Runner) getScanList() error {
items, err := r.cloudClient.GetScans()
loc, _ := time.LoadLocation("Local")
for _, v := range items {
status := "FINISHED"
t := v.FinishedAt
duration := t.Sub(v.CreatedAt)
if !v.Finished {
status = "RUNNING"
t = time.Now().UTC()
duration = t.Sub(v.CreatedAt)
}
val := v.CreatedAt.In(loc).Format(DDMMYYYYhhmmss)
gologger.Silent().Msgf("%s [%s] [STATUS: %s] [MATCHED: %d] [TARGETS: %d] [TEMPLATES: %d] [DURATION: %s]\n", v.Id, val, status, v.Matches, v.Targets, v.Templates, duration)
}
return err
}
func (r *Runner) deleteScan(id string) error {
deleted, err := r.cloudClient.DeleteScan(id)
if !deleted.OK {
gologger.Info().Msgf("Error in deleting the scan %s.", id)
} else {
gologger.Info().Msgf("Scan deleted %s.", id)
}
return err
}
func (r *Runner) getResults(id string) error {
err := r.cloudClient.GetResults(id, func(re *output.ResultEvent) {
if outputErr := r.output.Write(re); outputErr != nil {
gologger.Warning().Msgf("Could not write output: %s", outputErr)
}
}, false)
return err
}
// runCloudEnumeration runs cloud based enumeration
func (r *Runner) runCloudEnumeration(store *loader.Store) (*atomic.Bool, error) {
func (r *Runner) runCloudEnumeration(store *loader.Store, nostore bool) (*atomic.Bool, error) {
now := time.Now()
defer func() {
gologger.Info().Msgf("Scan execution took %s", time.Since(now))
}()
client := nucleicloud.New(r.options.CloudURL, r.options.CloudAPIKey)
results := &atomic.Bool{}
@ -41,9 +83,10 @@ func (r *Runner) runCloudEnumeration(store *loader.Store) (*atomic.Bool, error)
for _, template := range store.Templates() {
templates = append(templates, getTemplateRelativePath(template.Path))
}
taskID, err := client.AddScan(&nucleicloud.AddScanRequest{
taskID, err := r.cloudClient.AddScan(&nucleicloud.AddScanRequest{
RawTargets: targets,
PublicTemplates: templates,
IsTemporary: nostore,
})
if err != nil {
return results, err
@ -51,7 +94,7 @@ func (r *Runner) runCloudEnumeration(store *loader.Store) (*atomic.Bool, error)
gologger.Info().Msgf("Created task with ID: %s", taskID)
time.Sleep(3 * time.Second)
err = client.GetResults(taskID, func(re *output.ResultEvent) {
err = r.cloudClient.GetResults(taskID, func(re *output.ResultEvent) {
results.CompareAndSwap(false, true)
if outputErr := r.output.Write(re); outputErr != nil {
@ -62,7 +105,7 @@ func (r *Runner) runCloudEnumeration(store *loader.Store) (*atomic.Bool, error)
gologger.Warning().Msgf("Could not create issue on tracker: %s", err)
}
}
})
}, true)
return results, err
}

74
v2/internal/runner/nucleicloud/cloud.go
View File

@ -24,6 +24,7 @@ type Client struct {
const (
pollInterval = 1 * time.Second
defaultBaseURL = "http://webapp.localhost"
resultSize = 100
)
// New returns a nuclei-cloud API client
@ -72,10 +73,11 @@ func (c *Client) AddScan(req *AddScanRequest) (string, error) {
// GetResults gets results from nuclei server for an ID
// until there are no more results left to retrieve.
func (c *Client) GetResults(ID string, callback func(*output.ResultEvent)) error {
func (c *Client) GetResults(ID string, callback func(*output.ResultEvent), checkProgress bool) error {
lastID := int64(0)
for {
httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/results?id=%s&from=%d&size=100", c.baseURL, ID, lastID), nil)
uri := fmt.Sprintf("%s/results?id=%s&from=%d&size=%d", c.baseURL, ID, lastID, resultSize)
httpReq, err := retryablehttp.NewRequest(http.MethodGet, uri, nil)
if err != nil {
return errors.Wrap(err, "could not make request")
}
@ -106,10 +108,76 @@ func (c *Client) GetResults(ID string, callback func(*output.ResultEvent)) error
}
callback(&result)
}
if items.Finished && len(items.Items) == 0 {
//This is checked during scan is added else if no item found break out of loop.
if checkProgress {
if items.Finished && len(items.Items) == 0 {
break
}
} else if len(items.Items) == 0 {
break
}
time.Sleep(pollInterval)
}
return nil
}
func (c *Client) GetScans() ([]GetScanRequest, error) {
var items []GetScanRequest
httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/scan", c.baseURL), nil)
if err != nil {
return items, errors.Wrap(err, "could not make request")
}
httpReq.Header.Set("X-API-Key", c.apiKey)
resp, err := c.httpclient.Do(httpReq)
if err != nil {
return items, errors.Wrap(err, "could not make request.")
}
if err != nil {
return items, errors.Wrap(err, "could not do get response.")
}
if resp.StatusCode != 200 {
data, _ := io.ReadAll(resp.Body)
resp.Body.Close()
return items, errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data))
}
if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil {
resp.Body.Close()
return items, errors.Wrap(err, "could not decode results")
}
resp.Body.Close()
return items, nil
}
//Delete a scan and it's issues by the scan id.
func (c *Client) DeleteScan(id string) (DeleteScanResults, error) {
deletescan := DeleteScanResults{}
httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/scan?id=%s", c.baseURL, id), nil)
if err != nil {
return deletescan, errors.Wrap(err, "could not make request")
}
httpReq.Header.Set("X-API-Key", c.apiKey)
resp, err := c.httpclient.Do(httpReq)
if err != nil {
return deletescan, errors.Wrap(err, "could not make request")
}
if err != nil {
return deletescan, errors.Wrap(err, "could not do get result request")
}
if resp.StatusCode != 200 {
data, _ := io.ReadAll(resp.Body)
resp.Body.Close()
return deletescan, errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data))
}
if err := jsoniter.NewDecoder(resp.Body).Decode(&deletescan); err != nil {
resp.Body.Close()
return deletescan, errors.Wrap(err, "could not delete scan")
}
resp.Body.Close()
return deletescan, nil
}

19
v2/internal/runner/nucleicloud/types.go
View File

@ -1,5 +1,7 @@
package nucleicloud
import "time"
// AddScanRequest is a nuclei scan input item.
type AddScanRequest struct {
// RawTargets is a list of raw target URLs for the scan.
@ -9,6 +11,7 @@ type AddScanRequest struct {
// PrivateTemplates is a map of template-name->contents that
// are private to the user executing the scan. (TODO: TBD)
PrivateTemplates map[string]string `json:"private_templates,omitempty"`
IsTemporary bool `json:"is_temporary"`
}
type GetResultsResponse struct {
@ -16,7 +19,23 @@ type GetResultsResponse struct {
Items []GetResultsResponseItem `json:"items"`
}
type GetScanRequest struct {
Id string `json:"id"`
Total int32 `json:"total"`
Current int32 `json:"current"`
Finished bool `json:"finished"`
CreatedAt time.Time `json:"created_at"`
FinishedAt time.Time `json:"finished_at"`
Targets int32 `json:"targets"`
Templates int32 `json:"templates"`
Matches int64 `json:"matches"`
}
type GetResultsResponseItem struct {
ID int64 `json:"id"`
Raw string `json:"raw"`
}
type DeleteScanResults struct {
OK bool `json:"ok"`
}

26
v2/internal/runner/runner.go
View File

@ -14,6 +14,8 @@ import (
"strings"
"time"
"github.com/projectdiscovery/nuclei/v2/internal/runner/nucleicloud"
"github.com/blang/semver"
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
@ -70,6 +72,7 @@ type Runner struct {
hostErrors hosterrorscache.CacheInterface
resumeCfg *types.ResumeCfg
pprofServer *http.Server
cloudClient *nucleicloud.Client
}
const pprofServerAddress = "127.0.0.1:8086"
@ -85,6 +88,10 @@ func New(options *types.Options) (*Runner, error) {
os.Exit(0)
}
if options.Cloud {
runner.cloudClient = nucleicloud.New(options.CloudURL, options.CloudAPIKey)
}
if options.UpdateNuclei {
if err := updateNucleiVersionToLatest(runner.options.Verbose); err != nil {
return nil, err
@ -418,12 +425,27 @@ func (r *Runner) RunEnumeration() error {
executerOpts.InputHelper.InputsHTTP = inputHelpers
}
enumeration := false
var results *atomic.Bool
if r.options.Cloud {
gologger.Info().Msgf("Running scan on cloud with URL %s", r.options.CloudURL)
results, err = r.runCloudEnumeration(store)
if r.options.ScanList {
err = r.getScanList()
} else if r.options.DeleteScan != "" {
err = r.deleteScan(r.options.DeleteScan)
} else if r.options.ScanOutput != "" {
err = r.getResults(r.options.ScanOutput)
} else {
gologger.Info().Msgf("Running scan on cloud with URL %s", r.options.CloudURL)
results, err = r.runCloudEnumeration(store, r.options.NoStore)
enumeration = true
}
} else {
results, err = r.runStandardEnumeration(executerOpts, store, engine)
enumeration = true
}
if !enumeration {
return err
}
if r.interactsh != nil {

8
v2/pkg/types/types.go
View File

@ -95,6 +95,14 @@ type Options struct {
CloudURL string
// CloudAPIKey is the api-key for the nuclei cloud endpoint
CloudAPIKey string
// Scanlist feature to get all the scan ids for a user
ScanList bool
// Nostore
NoStore bool
// Delete scan
DeleteScan string
// Get issues for a scan
ScanOutput string
// ResolversFile is a file containing resolvers for nuclei.
ResolversFile string
// StatsInterval is the number of seconds to display stats after