daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cloud templates targets sync (#2959) 

* Add s3 bucket template provider

- Refactor the custom github template code
- add interface for template provider

* Validate if aws creds are passed if bucket flag

- refactor s3 provider struct to take client
- add function which returns the aws s3 client
- update error messages

* Add aws s3 bucket flags documentation in README.md

- Rename the github_test.go to customTemplate_test.go

* go mod update

* Move template provider code to pkg/external/customtemplates dir

* Added initial data_source sync to cloud

* Misc

* Add pagination to scan output and scan list (#2858)

* Add pagination to scan output and scan list

* Use time based parameters instead of page numbers

* Fix linting errors

* Do not check limits at client, check at server

* Remove unused constant

* Misc update

* Removed unnecessary flags

* Misc

* Misc

* Misc endpoint additions

* Added more routes

* Typo fix

* Misc fixes

* Misc

* Misc fixes to cloud target logic + use int for IDs

* Misc

* Misc fixes

* Misc

* Misc fixes

* readme update

* Add JSON output support for list-scan option (#2876)

* Add JSON output support for list-scan option

* Fix typo in cloud JSON output description

* Following changes

- Update status(finished, running) to be lower-case by default
- Convert status to upper-case in DisplayScanList()

* Update status to be lower-case by default

* Remove additional json flag, instead use existing

* Merge conflict

* Accomodate comment changes and restructure code

Co-authored-by: Jaideep K <jaideep@one2n.in>

* Use integer IDs for scan tasks

* Added get-templates-targets endpoint + JSON + validation

* Added target count list

* misc option / description updates

* Added changes as per code review

* duplicate options + typo updates

* Added tablewriter for tabular data writing by default

* Fixed list scan endpoint

* Review changes

* workflow fix

* Added cloud tags etc based filtering (#3070)

* Added omitempty for filtering request

* go mod tidy

* misc format update

Co-authored-by: shubhamrasal <shubhamdharmarasal@gmail.com>
Co-authored-by: Ice3man <nizamulrana@gmail.com>
Co-authored-by: Jaideep Khandelwal <jdk2588@gmail.com>
Co-authored-by: Siddharth Shashikar <60960197+shashikarsiddharth@users.noreply.github.com>
Co-authored-by: Jaideep K <jaideep@one2n.in>
dev
Sandeep Singh 2022-12-21 22:48:43 +05:30 committed by GitHub
parent 093d691c16
commit 96646c8f53
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 1196 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/publish-docs.yaml vendored
View File

@ -12,8 +12,6 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: "Set up Go"
uses: actions/setup-go@v3
@ -38,6 +36,7 @@ jobs:
run: |
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
git pull
git add SYNTAX-REFERENCE.md nuclei-jsonschema.json
git commit -m "Auto Generate Syntax Docs + JSONSchema [$(date)] :robot:" -a
@ -46,4 +45,4 @@ jobs:
uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: ${{ github.ref }}
branch: ${{ github.head_ref }}

23
v2/cmd/nuclei/main.go
View File

@ -295,12 +295,23 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.CreateGroup("cloud", "Cloud",
flagSet.BoolVar(&options.Cloud, "cloud", false, "run scan on nuclei cloud"),
flagSet.StringVarEnv(&options.CloudURL, "cloud-server", "cs", "https://cloud-dev.nuclei.sh", "NUCLEI_CLOUD_SERVER", "nuclei cloud server to use (NUCLEI_CLOUD_SERVER)"),
flagSet.StringVarEnv(&options.CloudAPIKey, "cloud-api-key", "ak", "", "NUCLEI_CLOUD_APIKEY", "api-key for the nuclei cloud server (NUCLEI_CLOUD_APIKEY)"),
flagSet.BoolVarP(&options.ScanList, "list-scan", "ls", false, "list previous cloud scans"),
flagSet.BoolVarP(&options.NoStore, "no-store", "ns", false, "disable scan/output storage on cloud"),
flagSet.StringVarP(&options.DeleteScan, "delete-scan", "ds", "", "delete scan/output on cloud by scan id"),
flagSet.StringVarP(&options.ScanOutput, "scan-output", "so", "", "display scan output by scan id"),
flagSet.StringVarP(&options.AddDatasource, "add-datasource", "ads", "", "add specified data source (s3,github)"),
flagSet.StringVarP(&options.AddTarget, "add-target", "atr", "", "add target(s) to cloud"),
flagSet.StringVarP(&options.AddTemplate, "add-template", "atm", "", "add template(s) to cloud"),
flagSet.BoolVarP(&options.ScanList, "list-scan", "lsn", false, "list previous cloud scans"),
flagSet.BoolVarP(&options.ListTargets, "list-target", "ltr", false, "list cloud target by id"),
flagSet.BoolVarP(&options.ListTemplates, "list-template", "ltm", false, "list cloud template by id"),
flagSet.BoolVarP(&options.ListDatasources, "list-datasource", "lds", false, "list cloud datasource by id"),
flagSet.StringVarP(&options.DeleteScan, "delete-scan", "dsn", "", "delete cloud scan by id"),
flagSet.StringVarP(&options.RemoveTarget, "delete-target", "dtr", "", "delete target(s) from cloud"),
flagSet.StringVarP(&options.RemoveTemplate, "delete-template", "dtm", "", "delete template(s) from cloud"),
flagSet.StringVarP(&options.RemoveDatasource, "delete-datasource", "dds", "", "delete specified data source"),
flagSet.StringVarP(&options.GetTarget, "get-target", "gtr", "", "get target content by id"),
flagSet.StringVarP(&options.GetTemplate, "get-template", "gtm", "", "get template content by id"),
flagSet.BoolVarP(&options.NoStore, "no-store", "nos", false, "disable scan/output storage on cloud"),
flagSet.StringVarP(&options.ScanOutput, "scan-output", "sno", "", "display scan output by scan id"),
flagSet.BoolVar(&options.NoTables, "no-tables", false, "do not display pretty-printed tables"),
flagSet.IntVar(&options.OutputLimit, "limit", 100, "limit the number of output to display"),
)
_ = flagSet.Parse()

18
v2/go.mod
View File

@ -30,7 +30,7 @@ require (
github.com/projectdiscovery/interactsh v1.0.6-0.20220827132222-460cc6270053
github.com/projectdiscovery/rawhttp v0.1.4
github.com/projectdiscovery/retryabledns v1.0.17
github.com/projectdiscovery/retryablehttp-go v1.0.5-0.20221202084821-c1a692a64751
github.com/projectdiscovery/retryablehttp-go v1.0.6-0.20221206071935-7924d7d34953
github.com/projectdiscovery/stringsutil v0.0.2
github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6
github.com/remeh/sizedwaitgroup v1.0.0
@ -87,7 +87,14 @@ require (
)
require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.3.1 // indirect
github.com/bits-and-blooms/bloom/v3 v3.3.1 // indirect
github.com/cloudflare/circl v1.1.0 // indirect
github.com/dlclark/regexp2 v1.4.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
@ -111,6 +118,7 @@ require (
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tidwall/rtred v0.1.2 // indirect
github.com/tidwall/tinyqueue v0.1.1 // indirect
github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 // indirect
gopkg.in/djherbis/times.v1 v1.3.0 // indirect
)
@ -126,8 +134,6 @@ require (
github.com/andybalholm/cascadia v1.1.0 // indirect
github.com/antchfx/xpath v1.2.1 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/bits-and-blooms/bitset v1.3.1 // indirect
github.com/bits-and-blooms/bloom/v3 v3.3.1 // indirect
github.com/c4milo/unpackit v0.1.0 // indirect
github.com/caddyserver/certmagic v0.16.3 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
@ -218,16 +224,11 @@ require (
github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect
github.com/acomagu/bufpipe v1.0.3 // indirect
github.com/alecthomas/chroma v0.10.0
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.20 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.27 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.20 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.26 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.9 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.17.6 // indirect
@ -250,6 +251,5 @@ require (
github.com/src-d/gcfg v1.4.0 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
)

4
v2/go.sum
View File

@ -637,8 +637,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.17 h1:XKzI26UKYt2g7YLJ/EcyYmM04sfD1vurETecPEpeA1w=
github.com/projectdiscovery/retryabledns v1.0.17/go.mod h1:Dyhq/f0sGmXueso0+Ah3LbJfsX4PXpBrpfiyjZZ8SDk=
github.com/projectdiscovery/retryablehttp-go v1.0.5-0.20221202084821-c1a692a64751 h1:QEmZ0E6GDzlTbVE6ty7fCuKR7muWrLqMfQ07VTu6Bd0=
github.com/projectdiscovery/retryablehttp-go v1.0.5-0.20221202084821-c1a692a64751/go.mod h1:B/xfvUmiJBeq+1kT7AMYL6B/IuPgbyKB7QCKPSfMByc=
github.com/projectdiscovery/retryablehttp-go v1.0.6-0.20221206071935-7924d7d34953 h1:8voTfrLQKt/nElO7zx6yoaFZDCeDq2xFHkWEl3fXBBI=
github.com/projectdiscovery/retryablehttp-go v1.0.6-0.20221206071935-7924d7d34953/go.mod h1:Z9FoXiCxITq+Wt6VzqJAu8xA/xDTVrgBFaDk+Xsfgw4=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/sliceutil v0.0.1 h1:YoCqCMcdwz+gqNfW5hFY8UvNHoA6SfyBSNkVahatleg=

319
v2/internal/runner/cloud.go Normal file
View File

@ -0,0 +1,319 @@
package runner
import (
"io"
"io/fs"
"os"
"path/filepath"
"strconv"
"strings"
jsoniter "github.com/json-iterator/go"
"github.com/olekukonko/tablewriter"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/runner/nucleicloud"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
)
// Get all the scan lists for a user/apikey.
func (r *Runner) getScanList(limit int) error {
lastTime := "2099-01-02 15:04:05 +0000 UTC"
header := []string{"ID", "Timestamp", "Targets", "Templates", "Matched", "Duration", "Status"}
var (
values [][]string
count int
)
for {
items, err := r.cloudClient.GetScans(limit, lastTime)
if err != nil {
return err
}
if len(items) == 0 {
break
}
for _, v := range items {
count++
lastTime = v.CreatedAt.String()
res := nucleicloud.PrepareScanListOutput(v)
if r.options.JSON {
_ = jsoniter.NewEncoder(os.Stdout).Encode(res)
} else if !r.options.NoTables {
values = append(values, []string{strconv.FormatInt(res.ScanID, 10), res.Timestamp, strconv.Itoa(res.Target), strconv.Itoa(res.Template), strconv.Itoa(res.ScanResult), res.ScanTime, res.ScanStatus})
} else {
gologger.Silent().Msgf("%d. [%s] [TARGETS: %d] [TEMPLATES: %d] [MATCHED: %d] [DURATION: %s] [STATUS: %s]\n", res.ScanID, res.Timestamp, res.Target, res.Template, res.ScanResult, res.ScanTime, strings.ToUpper(res.ScanStatus))
}
}
}
if count == 0 {
return errors.New("no scan found")
}
if !r.options.NoTables {
r.prettyPrintTable(header, values)
}
return nil
}
func (r *Runner) listDatasources() error {
datasources, err := r.cloudClient.ListDatasources()
if err != nil {
return err
}
if len(datasources) == 0 {
return errors.New("no cloud datasource found")
}
header := []string{"ID", "UpdatedAt", "Type", "Repo", "Path"}
var values [][]string
for _, source := range datasources {
if r.options.JSON {
_ = jsoniter.NewEncoder(os.Stdout).Encode(source)
} else if !r.options.NoTables {
values = append(values, []string{strconv.FormatInt(source.ID, 10), source.Updatedat.Format(nucleicloud.DDMMYYYYhhmmss), source.Type, source.Repo, source.Path})
} else {
gologger.Silent().Msgf("%d. [%s] [%s] [%s] %s", source.ID, source.Updatedat.Format(nucleicloud.DDMMYYYYhhmmss), source.Type, source.Repo, source.Path)
}
}
if !r.options.NoTables {
r.prettyPrintTable(header, values)
}
return nil
}
func (r *Runner) listTargets() error {
items, err := r.cloudClient.ListTargets("")
if err != nil {
return err
}
if len(items) == 0 {
return errors.New("no target found")
}
header := []string{"ID", "Reference", "Count"}
var values [][]string
for _, source := range items {
if r.options.JSON {
_ = jsoniter.NewEncoder(os.Stdout).Encode(source)
} else if !r.options.NoTables {
values = append(values, []string{strconv.FormatInt(source.ID, 10), source.Reference, strconv.FormatInt(source.Count, 10)})
} else {
gologger.Silent().Msgf("%d. %s (%d)", source.ID, source.Reference, source.Count)
}
}
if !r.options.NoTables {
r.prettyPrintTable(header, values)
}
return nil
}
func (r *Runner) listTemplates() error {
items, err := r.cloudClient.ListTemplates("")
if err != nil {
return err
}
if len(items) == 0 {
return errors.New("no template found")
}
header := []string{"ID", "Reference"}
var values [][]string
for _, source := range items {
if r.options.JSON {
_ = jsoniter.NewEncoder(os.Stdout).Encode(source)
} else if !r.options.NoTables {
values = append(values, []string{strconv.FormatInt(source.ID, 10), source.Reference})
} else {
gologger.Silent().Msgf("%d. %s", source.ID, source.Reference)
}
}
if !r.options.NoTables {
r.prettyPrintTable(header, values)
}
return nil
}
func (r *Runner) prettyPrintTable(header []string, values [][]string) {
writer := tablewriter.NewWriter(os.Stdout)
writer.SetHeader(header)
writer.AppendBulk(values)
writer.Render()
}
func (r *Runner) deleteScan(id string) error {
ID, _ := strconv.ParseInt(id, 10, 64)
deleted, err := r.cloudClient.DeleteScan(ID)
if !deleted.OK {
gologger.Error().Msgf("Error in deleting the scan %s.", id)
} else {
gologger.Info().Msgf("Scan deleted %s.", id)
}
return err
}
func (r *Runner) getResults(id string, limit int) error {
ID, _ := strconv.ParseInt(id, 10, 64)
err := r.cloudClient.GetResults(ID, false, limit, func(re *output.ResultEvent) {
if outputErr := r.output.Write(re); outputErr != nil {
gologger.Warning().Msgf("Could not write output: %s", outputErr)
}
})
return err
}
func (r *Runner) getTarget(id string) error {
ID, _ := strconv.ParseInt(id, 10, 64)
reader, err := r.cloudClient.GetTarget(ID)
if err != nil {
return errors.Wrap(err, "could not get target")
}
defer reader.Close()
_, _ = io.Copy(os.Stdout, reader)
return nil
}
func (r *Runner) getTemplate(id string) error {
ID, _ := strconv.ParseInt(id, 10, 64)
reader, err := r.cloudClient.GetTemplate(ID)
if err != nil {
return errors.Wrap(err, "could not get template")
}
defer reader.Close()
_, _ = io.Copy(os.Stdout, reader)
return nil
}
func (r *Runner) removeDatasource(datasource string) error {
var source string
ID, parseErr := strconv.ParseInt(datasource, 10, 64)
if parseErr != nil {
source = datasource
}
err := r.cloudClient.RemoveDatasource(ID, source)
if err == nil {
gologger.Info().Msgf("Datasource deleted %s", datasource)
}
return err
}
func (r *Runner) addTemplate(location string) error {
walkErr := filepath.WalkDir(location, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".yaml") {
return nil
}
base := filepath.Base(path)
reference, templateErr := r.cloudClient.AddTemplate(base, path)
if templateErr != nil {
gologger.Error().Msgf("Could not upload %s: %s", path, templateErr)
} else if reference != "" {
gologger.Info().Msgf("Uploaded template %s: %s", base, reference)
}
return nil
})
return walkErr
}
func (r *Runner) addTarget(location string) error {
walkErr := filepath.WalkDir(location, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".txt") {
return nil
}
base := filepath.Base(location)
reference, targetErr := r.cloudClient.AddTarget(base, location)
if targetErr != nil {
gologger.Error().Msgf("Could not upload %s: %s", location, targetErr)
} else if reference != "" {
gologger.Info().Msgf("Uploaded target %s: %s", base, reference)
}
return nil
})
return walkErr
}
func (r *Runner) removeTarget(item string) error {
response, err := r.cloudClient.ListTargets(item)
if err != nil {
return errors.Wrap(err, "could not list targets")
}
for _, item := range response {
if err := r.cloudClient.RemoveTarget(item.ID); err != nil {
gologger.Error().Msgf("Error in deleting target %s: %s", item.Reference, err)
} else {
gologger.Info().Msgf("Target deleted %s", item.Reference)
}
}
return nil
}
func (r *Runner) removeTemplate(item string) error {
response, err := r.cloudClient.ListTemplates(item)
if err != nil {
return errors.Wrap(err, "could not list templates")
}
for _, item := range response {
if err := r.cloudClient.RemoveTemplate(item.ID); err != nil {
gologger.Error().Msgf("Error in deleting template %s: %s", item.Reference, err)
} else {
gologger.Info().Msgf("Template deleted %s", item.Reference)
}
}
return nil
}
// initializeCloudDataSources initializes cloud data sources
func (r *Runner) addCloudDataSource(source string) error {
switch source {
case "s3":
token := strings.Join([]string{r.options.AwsAccessKey, r.options.AwsSecretKey, r.options.AwsRegion}, ":")
if _, err := r.processDataSourceItem(r.options.AwsBucketName, token, "s3"); err != nil {
return err
}
case "github":
for _, repo := range r.options.GithubTemplateRepo {
if _, err := r.processDataSourceItem(repo, r.options.GithubToken, "github"); err != nil {
return err
}
}
}
return nil
}
func (r *Runner) processDataSourceItem(repo, token, Type string) (int64, error) {
ID, err := r.cloudClient.StatusDataSource(nucleicloud.StatusDataSourceRequest{Repo: repo, Token: token})
if err != nil {
if !strings.Contains(err.Error(), "no rows in result set") {
return 0, errors.Wrap(err, "could not get data source status")
}
gologger.Info().Msgf("Adding new data source + syncing: %s\n", repo)
resp, err := r.cloudClient.AddDataSource(nucleicloud.AddDataSourceRequest{Type: Type, Repo: repo, Token: token})
if err != nil {
return 0, errors.Wrap(err, "could not add data source")
}
ID = resp.ID
if err = r.cloudClient.SyncDataSource(resp.ID); err != nil {
return 0, errors.Wrap(err, "could not sync data source")
}
if resp.Secret != "" {
gologger.Info().Msgf("Webhook URL for added source: %s/datasources/%s/webhook", r.options.CloudURL, resp.Hash)
gologger.Info().Msgf("Secret for webhook: %s", resp.Secret)
}
}
if r.options.UpdateTemplates {
gologger.Info().Msgf("Syncing data source: %s (%d)\n", repo, ID)
if err = r.cloudClient.SyncDataSource(ID); err != nil {
return 0, errors.Wrap(err, "could not sync data source")
}
}
return ID, nil
}

81
v2/internal/runner/enumerate.go
View File

@ -20,11 +20,10 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"go.uber.org/atomic"
)
const DDMMYYYYhhmmss = "2006-01-02 15:04:05"
// runStandardEnumeration runs standard enumeration
func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions, store *loader.Store, engine *core.Engine) (*atomic.Bool, error) {
if r.options.AutomaticScan {
@ -33,52 +32,12 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions,
return r.executeTemplatesInput(store, engine)
}
// Get all the scan lists for a user/apikey.
func (r *Runner) getScanList() error {
items, err := r.cloudClient.GetScans()
loc, _ := time.LoadLocation("Local")
for _, v := range items {
status := "FINISHED"
t := v.FinishedAt
duration := t.Sub(v.CreatedAt)
if !v.Finished {
status = "RUNNING"
t = time.Now().UTC()
duration = t.Sub(v.CreatedAt).Round(60 * time.Second)
}
val := v.CreatedAt.In(loc).Format(DDMMYYYYhhmmss)
gologger.Silent().Msgf("%s [%s] [STATUS: %s] [MATCHED: %d] [TARGETS: %d] [TEMPLATES: %d] [DURATION: %s]\n", v.Id, val, status, v.Matches, v.Targets, v.Templates, duration)
}
return err
}
func (r *Runner) deleteScan(id string) error {
deleted, err := r.cloudClient.DeleteScan(id)
if !deleted.OK {
gologger.Info().Msgf("Error in deleting the scan %s.", id)
} else {
gologger.Info().Msgf("Scan deleted %s.", id)
}
return err
}
func (r *Runner) getResults(id string) error {
err := r.cloudClient.GetResults(id, func(re *output.ResultEvent) {
if outputErr := r.output.Write(re); outputErr != nil {
gologger.Warning().Msgf("Could not write output: %s", outputErr)
}
}, false)
return err
}
// runCloudEnumeration runs cloud based enumeration
func (r *Runner) runCloudEnumeration(store *loader.Store, nostore bool) (*atomic.Bool, error) {
func (r *Runner) runCloudEnumeration(store *loader.Store, cloudTemplates, cloudTargets []string, nostore bool, limit int) (*atomic.Bool, error) {
count := &atomic.Int64{}
now := time.Now()
defer func() {
gologger.Info().Msgf("Scan execution took %s", time.Since(now))
gologger.Info().Msgf("Scan execution took %s and found %d results", time.Since(now), count.Load())
}()
results := &atomic.Bool{}
@ -110,17 +69,24 @@ func (r *Runner) runCloudEnumeration(store *loader.Store, nostore bool) (*atomic
taskID, err := r.cloudClient.AddScan(&nucleicloud.AddScanRequest{
RawTargets: targets,
PublicTemplates: templates,
CloudTargets: cloudTargets,
CloudTemplates: cloudTemplates,
PrivateTemplates: privateTemplates,
IsTemporary: nostore,
Filtering: getCloudFilteringFromOptions(r.options),
})
if err != nil {
return results, err
}
gologger.Info().Msgf("Created task with ID: %s", taskID)
gologger.Info().Msgf("Created task with ID: %d", taskID)
if nostore {
gologger.Info().Msgf("Cloud scan storage: disabled")
}
time.Sleep(3 * time.Second)
err = r.cloudClient.GetResults(taskID, func(re *output.ResultEvent) {
err = r.cloudClient.GetResults(taskID, true, limit, func(re *output.ResultEvent) {
results.CompareAndSwap(false, true)
_ = count.Inc()
if outputErr := r.output.Write(re); outputErr != nil {
gologger.Warning().Msgf("Could not write output: %s", err)
@ -130,7 +96,7 @@ func (r *Runner) runCloudEnumeration(store *loader.Store, nostore bool) (*atomic
gologger.Warning().Msgf("Could not create issue on tracker: %s", err)
}
}
}, true)
})
return results, err
}
@ -150,3 +116,22 @@ func gzipBase64EncodeData(data []byte) string {
encoded := base64.StdEncoding.EncodeToString(buf.Bytes())
return encoded
}
func getCloudFilteringFromOptions(options *types.Options) *nucleicloud.AddScanRequestConfiguration {
return &nucleicloud.AddScanRequestConfiguration{
Authors: options.Authors,
Tags: options.Tags,
ExcludeTags: options.ExcludeTags,
IncludeTags: options.IncludeTags,
IncludeIds: options.IncludeIds,
ExcludeIds: options.ExcludeIds,
IncludeTemplates: options.IncludeTemplates,
ExcludedTemplates: options.ExcludedTemplates,
ExcludeMatchers: options.ExcludeMatchers,
Severities: options.Severities,
ExcludeSeverities: options.ExcludeSeverities,
Protocols: options.Protocols,
ExcludeProtocols: options.ExcludeProtocols,
IncludeConditions: options.IncludeConditions,
}
}

465
v2/internal/runner/nucleicloud/cloud.go
View File

@ -3,9 +3,15 @@ package nucleicloud
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
@ -23,9 +29,9 @@ type Client struct {
}
const (
pollInterval = 1 * time.Second
defaultBaseURL = "http://webapp.localhost"
pollInterval = 3 * time.Second
resultSize = 100
defaultBaseURL = "https://cloud-dev.nuclei.sh"
)
// HTTPErrorRetryPolicy is to retry for HTTPCodes >= 500.
@ -41,7 +47,8 @@ func HTTPErrorRetryPolicy() func(ctx context.Context, resp *http.Response, err e
// New returns a nuclei-cloud API client
func New(baseURL, apiKey string) *Client {
options := retryablehttp.DefaultOptionsSingle
options.Timeout = 15 * time.Second
options.NoAdjustTimeout = true
options.Timeout = 60 * time.Second
options.CheckRetry = HTTPErrorRetryPolicy()
client := retryablehttp.NewClient(options)
@ -53,57 +60,47 @@ func New(baseURL, apiKey string) *Client {
}
// AddScan adds a scan for templates and target to nuclei server
func (c *Client) AddScan(req *AddScanRequest) (string, error) {
func (c *Client) AddScan(req *AddScanRequest) (int64, error) {
var buf bytes.Buffer
if err := jsoniter.NewEncoder(&buf).Encode(req); err != nil {
return "", errors.Wrap(err, "could not json encode scan request")
return 0, errors.Wrap(err, "could not encode request")
}
httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/scan", c.baseURL), bytes.NewReader(buf.Bytes()))
if err != nil {
return "", errors.Wrap(err, "could not make request")
return 0, errors.Wrap(err, "could not make request")
}
httpReq.Header.Set("X-API-Key", c.apiKey)
resp, err := c.httpclient.Do(httpReq)
resp, err := c.sendRequest(httpReq)
if err != nil {
return "", errors.Wrap(err, "could not do add scan request")
return 0, errors.Wrap(err, "could not do request")
}
if resp.StatusCode != 200 {
data, _ := io.ReadAll(resp.Body)
resp.Body.Close()
return "", errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data))
}
var data map[string]string
defer resp.Body.Close()
var data map[string]int64
if err := jsoniter.NewDecoder(resp.Body).Decode(&data); err != nil {
resp.Body.Close()
return "", errors.Wrap(err, "could not decode resp")
return 0, errors.Wrap(err, "could not decode resp")
}
resp.Body.Close()
id := data["id"]
return id, nil
}
// GetResults gets results from nuclei server for an ID
// until there are no more results left to retrieve.
func (c *Client) GetResults(ID string, callback func(*output.ResultEvent), checkProgress bool) error {
func (c *Client) GetResults(ID int64, checkProgress bool, limit int, callback func(*output.ResultEvent)) error {
lastID := int64(0)
for {
uri := fmt.Sprintf("%s/results?id=%s&from=%d&size=%d", c.baseURL, ID, lastID, resultSize)
uri := fmt.Sprintf("%s/results?id=%d&from=%d&size=%d", c.baseURL, ID, lastID, limit)
httpReq, err := retryablehttp.NewRequest(http.MethodGet, uri, nil)
if err != nil {
return errors.Wrap(err, "could not make request")
}
httpReq.Header.Set("X-API-Key", c.apiKey)
resp, err := c.httpclient.Do(httpReq)
resp, err := c.sendRequest(httpReq)
if err != nil {
return errors.Wrap(err, "could not do ger result request")
}
if resp.StatusCode != 200 {
data, _ := io.ReadAll(resp.Body)
resp.Body.Close()
return errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data))
return errors.Wrap(err, "could not do request")
}
var items GetResultsResponse
if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil {
resp.Body.Close()
@ -135,61 +132,409 @@ func (c *Client) GetResults(ID string, callback func(*output.ResultEvent), check
return nil
}
func (c *Client) GetScans() ([]GetScanRequest, error) {
func (c *Client) GetScans(limit int, from string) ([]GetScanRequest, error) {
var items []GetScanRequest
httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/scan", c.baseURL), nil)
httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/scan?from=%s&size=%d", c.baseURL, url.QueryEscape(from), limit), nil)
if err != nil {
return items, errors.Wrap(err, "could not make request")
}
httpReq.Header.Set("X-API-Key", c.apiKey)
resp, err := c.httpclient.Do(httpReq)
resp, err := c.sendRequest(httpReq)
if err != nil {
return items, errors.Wrap(err, "could not make request.")
}
if err != nil {
return items, errors.Wrap(err, "could not do get response.")
}
if resp.StatusCode != 200 {
data, _ := io.ReadAll(resp.Body)
resp.Body.Close()
return items, errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data))
return nil, errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil {
resp.Body.Close()
return items, errors.Wrap(err, "could not decode results")
}
resp.Body.Close()
return items, nil
}
// Delete a scan and it's issues by the scan id.
func (c *Client) DeleteScan(id string) (DeleteScanResults, error) {
func (c *Client) DeleteScan(id int64) (DeleteScanResults, error) {
deletescan := DeleteScanResults{}
httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/scan?id=%s", c.baseURL, id), nil)
httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/scan?id=%d", c.baseURL, id), nil)
if err != nil {
return deletescan, errors.Wrap(err, "could not make request")
}
httpReq.Header.Set("X-API-Key", c.apiKey)
resp, err := c.httpclient.Do(httpReq)
resp, err := c.sendRequest(httpReq)
if err != nil {
return deletescan, errors.Wrap(err, "could not make request")
}
if err != nil {
return deletescan, errors.Wrap(err, "could not do get result request")
}
if resp.StatusCode != 200 {
data, _ := io.ReadAll(resp.Body)
resp.Body.Close()
return deletescan, errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data))
return deletescan, errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
if err := jsoniter.NewDecoder(resp.Body).Decode(&deletescan); err != nil {
resp.Body.Close()
return deletescan, errors.Wrap(err, "could not delete scan")
}
resp.Body.Close()
return deletescan, nil
}
// StatusDataSource returns the status for a data source
func (c *Client) StatusDataSource(statusRequest StatusDataSourceRequest) (int64, error) {
var buf bytes.Buffer
if err := jsoniter.NewEncoder(&buf).Encode(statusRequest); err != nil {
return 0, errors.Wrap(err, "could not encode request")
}
httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/datasources/status", c.baseURL), bytes.NewReader(buf.Bytes()))
if err != nil {
return 0, errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return 0, errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
var data StatusDataSourceResponse
if err := jsoniter.NewDecoder(resp.Body).Decode(&data); err != nil {
return 0, errors.Wrap(err, "could not decode resp")
}
return data.ID, nil
}
// AddDataSource adds a new data source
func (c *Client) AddDataSource(req AddDataSourceRequest) (*AddDataSourceResponse, error) {
var buf bytes.Buffer
if err := jsoniter.NewEncoder(&buf).Encode(req); err != nil {
return nil, errors.Wrap(err, "could not encode request")
}
httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/datasources", c.baseURL), bytes.NewReader(buf.Bytes()))
if err != nil {
return nil, errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return nil, errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
var data AddDataSourceResponse
if err := jsoniter.NewDecoder(resp.Body).Decode(&data); err != nil {
return nil, errors.Wrap(err, "could not decode resp")
}
return &data, nil
}
// SyncDataSource syncs contents for a data source. The call blocks until
// update is completed.
func (c *Client) SyncDataSource(ID int64) error {
httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/datasources/%d/sync", c.baseURL, ID), nil)
if err != nil {
return errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, resp.Body)
return nil
}
// ExistsDataSourceItem identifies whether data source item exist
func (c *Client) ExistsDataSourceItem(req ExistsDataSourceItemRequest) error {
var buf bytes.Buffer
if err := jsoniter.NewEncoder(&buf).Encode(req); err != nil {
return errors.Wrap(err, "could not encode request")
}
httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/datasources/exists", c.baseURL), bytes.NewReader(buf.Bytes()))
if err != nil {
return errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, resp.Body)
return nil
}
func (c *Client) ListDatasources() ([]GetDataSourceResponse, error) {
var items []GetDataSourceResponse
httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/datasources", c.baseURL), nil)
if err != nil {
return items, errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return nil, errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil {
return items, errors.Wrap(err, "could not decode results")
}
return items, nil
}
func (c *Client) ListTargets(query string) ([]GetTargetResponse, error) {
var builder strings.Builder
_, _ = builder.WriteString(c.baseURL)
_, _ = builder.WriteString("/targets")
if query != "" {
_, _ = builder.WriteString("?query=")
_, _ = builder.WriteString(url.QueryEscape(query))
}
var items []GetTargetResponse
httpReq, err := retryablehttp.NewRequest(http.MethodGet, builder.String(), nil)
if err != nil {
return items, errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return nil, errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil {
return items, errors.Wrap(err, "could not decode results")
}
return items, nil
}
func (c *Client) ListTemplates(query string) ([]GetTemplatesResponse, error) {
var builder strings.Builder
_, _ = builder.WriteString(c.baseURL)
_, _ = builder.WriteString("/templates")
if query != "" {
_, _ = builder.WriteString("?query=")
_, _ = builder.WriteString(url.QueryEscape(query))
}
var items []GetTemplatesResponse
httpReq, err := retryablehttp.NewRequest(http.MethodGet, builder.String(), nil)
if err != nil {
return items, errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return nil, errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil {
return items, errors.Wrap(err, "could not decode results")
}
return items, nil
}
func (c *Client) RemoveDatasource(datasource int64, name string) error {
var builder strings.Builder
_, _ = builder.WriteString(c.baseURL)
_, _ = builder.WriteString("/datasources")
if name != "" {
_, _ = builder.WriteString("?name=")
_, _ = builder.WriteString(name)
} else if datasource != 0 {
_, _ = builder.WriteString("?id=")
_, _ = builder.WriteString(strconv.FormatInt(datasource, 10))
}
httpReq, err := retryablehttp.NewRequest(http.MethodDelete, builder.String(), nil)
if err != nil {
return errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, resp.Body)
return nil
}
func (c *Client) AddTemplate(name, contents string) (string, error) {
file, err := os.Open(contents)
if err != nil {
return "", errors.Wrap(err, "could not open contents")
}
defer file.Close()
var buf bytes.Buffer
writer := multipart.NewWriter(&buf)
_ = writer.WriteField("name", name)
fileWriter, _ := writer.CreateFormFile("file", filepath.Base(contents))
_, _ = io.Copy(fileWriter, file)
_ = writer.Close()
httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/templates", c.baseURL), &buf)
if err != nil {
return "", errors.Wrap(err, "could not make request")
}
httpReq.Header.Set("Content-Type", writer.FormDataContentType())
resp, err := c.sendRequest(httpReq)
if err != nil {
return "", errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
var item AddItemResponse
if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil {
return "", errors.Wrap(err, "could not decode results")
}
return item.Ok, nil
}
func (c *Client) AddTarget(name, contents string) (string, error) {
file, err := os.Open(contents)
if err != nil {
return "", errors.Wrap(err, "could not open contents")
}
defer file.Close()
var buf bytes.Buffer
writer := multipart.NewWriter(&buf)
_ = writer.WriteField("name", name)
fileWriter, _ := writer.CreateFormFile("file", filepath.Base(contents))
_, _ = io.Copy(fileWriter, file)
_ = writer.Close()
httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/targets", c.baseURL), &buf)
if err != nil {
return "", errors.Wrap(err, "could not make request")
}
httpReq.Header.Set("Content-Type", writer.FormDataContentType())
resp, err := c.sendRequest(httpReq)
if err != nil {
return "", errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
var item AddItemResponse
if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil {
return "", errors.Wrap(err, "could not decode results")
}
return item.Ok, nil
}
func (c *Client) RemoveTemplate(ID int64) error {
httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/templates/%d", c.baseURL, ID), nil)
if err != nil {
return errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, resp.Body)
return nil
}
func (c *Client) RemoveTarget(ID int64) error {
httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/targets/%d", c.baseURL, ID), nil)
if err != nil {
return errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, resp.Body)
return nil
}
func (c *Client) GetTarget(ID int64) (io.ReadCloser, error) {
httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/targets/%d", c.baseURL, ID), nil)
if err != nil {
return nil, errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return nil, errors.Wrap(err, "could not do request")
}
return resp.Body, nil
}
func (c *Client) GetTemplate(ID int64) (io.ReadCloser, error) {
httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/templates/%d", c.baseURL, ID), nil)
if err != nil {
return nil, errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return nil, errors.Wrap(err, "could not do request")
}
return resp.Body, nil
}
func (c *Client) ExistsTarget(id int64) (ExistsInputResponse, error) {
var item ExistsInputResponse
httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/targets/%d/exists", c.baseURL, id), nil)
if err != nil {
return item, errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return item, errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil {
return item, errors.Wrap(err, "could not decode results")
}
return item, nil
}
func (c *Client) ExistsTemplate(id int64) (ExistsInputResponse, error) {
var item ExistsInputResponse
httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/templates/%d/exists", c.baseURL, id), nil)
if err != nil {
return item, errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return item, errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil {
return item, errors.Wrap(err, "could not decode results")
}
return item, nil
}
const apiKeyParameter = "X-API-Key"
type errorResponse struct {
Message string `json:"message"`
}
func (c *Client) sendRequest(req *retryablehttp.Request) (*http.Response, error) {
req.Header.Set(apiKeyParameter, c.apiKey)
resp, err := c.httpclient.Do(req)
if err != nil {
return nil, errors.Wrap(err, "could not do request")
}
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest {
data, _ := io.ReadAll(resp.Body)
resp.Body.Close()
var errRes errorResponse
if err = json.NewDecoder(bytes.NewReader(data)).Decode(&errRes); err == nil {
return nil, errors.New(errRes.Message)
}
return nil, fmt.Errorf("unknown error, status code: %d=%s", resp.StatusCode, string(data))
}
return resp, nil
}

113
v2/internal/runner/nucleicloud/types.go
View File

@ -2,6 +2,9 @@ package nucleicloud
import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
)
// AddScanRequest is a nuclei scan input item.
@ -13,16 +16,41 @@ type AddScanRequest struct {
// PrivateTemplates is a map of template-name->contents that
// are private to the user executing the scan. (TODO: TBD)
PrivateTemplates map[string]string `json:"private_templates,omitempty"`
// CloudTargets is a list of cloud targets for the scan
CloudTargets []string `json:"cloud_targets,omitempty"`
// CloudTemplates is a list of cloud templates for the scan
CloudTemplates []string `json:"cloud_templates,omitempty"`
// Filtering contains optional filtering options for scan additions
Filtering *AddScanRequestConfiguration `json:"filtering"`
IsTemporary bool `json:"is_temporary"`
}
// AddScanRequestConfiguration contains filtering options for scan addition
type AddScanRequestConfiguration struct {
Authors []string `json:"author,omitempty"`
Tags []string `json:"tags,omitempty"`
ExcludeTags []string `json:"exclude-tags,omitempty"`
IncludeTags []string `json:"include-tags,omitempty"`
IncludeIds []string `json:"include-ids,omitempty"`
ExcludeIds []string `json:"exclude-ids,omitempty"`
IncludeTemplates []string `json:"include-templates,omitempty"`
ExcludedTemplates []string `json:"exclude-templates,omitempty"`
ExcludeMatchers []string `json:"exclude-matchers,omitempty"`
Severities severity.Severities `json:"severities,omitempty"`
ExcludeSeverities severity.Severities `json:"exclude-severities,omitempty"`
Protocols types.ProtocolTypes `json:"protocols,omitempty"`
ExcludeProtocols types.ProtocolTypes `json:"exclude-protocols,omitempty"`
IncludeConditions []string `json:"include-conditions,omitempty"`
}
type GetResultsResponse struct {
Finished bool `json:"finished"`
Items []GetResultsResponseItem `json:"items"`
}
type GetScanRequest struct {
Id string `json:"id"`
Id int64 `json:"id"`
Total int32 `json:"total"`
Current int32 `json:"current"`
Finished bool `json:"finished"`
@ -33,6 +61,13 @@ type GetScanRequest struct {
Matches int64 `json:"matches"`
}
// AddDataSourceResponse is a add data source response item.
type AddDataSourceResponse struct {
ID int64 `json:"id"`
Hash string `json:"hash"`
Secret string `json:"secret,omitempty"`
}
type GetResultsResponseItem struct {
ID int64 `json:"id"`
Raw string `json:"raw"`
@ -41,3 +76,79 @@ type GetResultsResponseItem struct {
type DeleteScanResults struct {
OK bool `json:"ok"`
}
// StatusDataSourceRequest is a add data source request item.
type StatusDataSourceRequest struct {
Repo string `json:"repo"`
Token string `json:"token"`
}
// StatusDataSourceResponse is a add data source response item.
type StatusDataSourceResponse struct {
ID int64 `json:"id"`
}
// AddDataSourceRequest is a add data source request item.
type AddDataSourceRequest struct {
Type string `json:"type"`
Repo string `json:"repo"`
Token string `json:"token"`
Sync bool `json:"sync"`
}
// ExistsDataSourceItemRequest is a request to identify whether a data
// source item exists.
type ExistsDataSourceItemRequest struct {
Type string `json:"type"`
Contents string `json:"contents"`
}
// GetDataSourceResponse is response for a get data source request
type GetDataSourceResponse struct {
ID int64 `json:"id"`
Hash string `json:"hash"`
Type string `json:"type"`
Path string `json:"path"`
Repo string `json:"repo"`
Updatedat time.Time `json:"updated_at"`
}
// GetTargetResponse is the response for a get target request
type GetTargetResponse struct {
ID int64 `json:"id"`
DataSource int64 `json:"data_source"`
Name string `json:"name"`
Reference string `json:"reference"`
Count int64 `json:"count"`
Hash string `json:"hash"`
Type string `json:"type"`
}
// GetTemplatesResponse is the response for a get templates request
type GetTemplatesResponse struct {
ID int64 `json:"id,omitempty"`
DataSource int64 `json:"data_source,omitempty"`
Name string `json:"name,omitempty"`
Reference string `json:"reference,omitempty"`
Hash string `json:"hash,omitempty"`
Type string `json:"type,omitempty"`
}
// AddItemResponse is the response to add item request
type AddItemResponse struct {
Ok string `json:"ok"`
}
type ListScanOutput struct {
Timestamp string `json:"timestamp"`
ScanID int64 `json:"scan_id"`
ScanTime string `json:"scan_time"`
ScanResult int `json:"scan_result"`
ScanStatus string `json:"scan_status"`
Target int `json:"target"`
Template int `json:"template"`
}
type ExistsInputResponse struct {
Reference string `json:"reference"`
}

29
v2/internal/runner/nucleicloud/utils.go
View File

@ -5,10 +5,13 @@ import (
"os"
"path/filepath"
"strings"
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
)
const DDMMYYYYhhmmss = "2006-01-02 15:04:05"
// ReadCatalogChecksum reads catalog checksum from nuclei-templates repository
func ReadCatalogChecksum() map[string]string {
config, _ := config.ReadConfiguration()
@ -38,3 +41,29 @@ func ReadCatalogChecksum() map[string]string {
}
return checksums
}
func PrepareScanListOutput(v GetScanRequest) ListScanOutput {
output := ListScanOutput{}
loc, _ := time.LoadLocation("Local")
status := "finished"
t := v.FinishedAt
duration := t.Sub(v.CreatedAt)
if !v.Finished {
status = "running"
t = time.Now().UTC()
duration = t.Sub(v.CreatedAt).Round(60 * time.Second)
}
val := v.CreatedAt.In(loc).Format(DDMMYYYYhhmmss)
output.Timestamp = val
output.ScanID = v.Id
output.ScanTime = duration.String()
output.ScanResult = int(v.Matches)
output.ScanStatus = status
output.Target = int(v.Targets)
output.Template = int(v.Templates)
return output
}

73
v2/internal/runner/options.go
View File

@ -160,16 +160,7 @@ func validateOptions(options *types.Options) error {
}
// Verify aws secrets are passed if s3 template bucket passed
if options.AwsBucketName != "" && options.UpdateTemplates {
var missing []string
if options.AwsAccessKey == "" {
missing = append(missing, "AWS_ACCESS_KEY")
}
if options.AwsSecretKey == "" {
missing = append(missing, "AWS_SECRET_KEY")
}
if options.AwsRegion == "" {
missing = append(missing, "AWS_REGION")
}
missing := validateMissingS3Options(options)
if missing != nil {
return fmt.Errorf("aws s3 bucket details are missing. Please provide %s", strings.Join(missing, ","))
}
@ -195,9 +186,63 @@ func validateOptions(options *types.Options) error {
return errors.New("ipv4 and/or ipv6 must be selected")
}
// Validate cloud option
if err := validateCloudOptions(options); err != nil {
return err
}
return nil
}
func validateCloudOptions(options *types.Options) error {
if options.HasCloudOptions() && !options.Cloud {
return errors.New("cloud flags cannot be used without cloud option")
}
if options.Cloud {
if options.CloudAPIKey == "" {
return errors.New("missing NUCLEI_CLOUD_API env variable")
}
var missing []string
switch options.AddDatasource {
case "s3":
missing = validateMissingS3Options(options)
case "github":
missing = validateMissingGithubOptions(options)
}
if len(missing) > 0 {
return fmt.Errorf("missing %v env variables", strings.Join(missing, ", "))
}
}
return nil
}
func validateMissingS3Options(options *types.Options) []string {
var missing []string
if options.AwsBucketName == "" {
missing = append(missing, "AWS_TEMPLATE_BUCKET")
}
if options.AwsAccessKey == "" {
missing = append(missing, "AWS_ACCESS_KEY")
}
if options.AwsSecretKey == "" {
missing = append(missing, "AWS_SECRET_KEY")
}
if options.AwsRegion == "" {
missing = append(missing, "AWS_REGION")
}
return missing
}
func validateMissingGithubOptions(options *types.Options) []string {
var missing []string
if options.GithubToken == "" {
missing = append(missing, "GITHUB_TOKEN")
}
if len(options.GithubTemplateRepo) == 0 {
missing = append(missing, "GITHUB_TEMPLATE_REPO")
}
return missing
}
// configureOutput configures the output logging levels to be displayed on the screen
func configureOutput(options *types.Options) {
// If the user desires verbose output, show verbose output
@ -275,6 +320,14 @@ func validateCertificatePaths(certificatePaths []string) {
// Read the input from env and set options
func readEnvInputVars(options *types.Options) {
if strings.EqualFold(os.Getenv("NUCLEI_CLOUD"), "true") {
options.Cloud = true
}
if options.CloudURL = os.Getenv("NUCLEI_CLOUD_SERVER"); options.CloudURL == "" {
options.CloudURL = "https://cloud-dev.nuclei.sh"
}
options.CloudAPIKey = os.Getenv("NUCLEI_CLOUD_API")
options.GithubToken = os.Getenv("GITHUB_TOKEN")
repolist := os.Getenv("GITHUB_TEMPLATE_REPO")
if repolist != "" {

74
v2/internal/runner/runner.go
View File

@ -11,6 +11,7 @@ import (
_ "net/http/pprof"
"os"
"path/filepath"
"strconv"
"strings"
"time"
@ -77,6 +78,7 @@ type Runner struct {
pprofServer *http.Server
customTemplates []customtemplates.Provider
cloudClient *nucleicloud.Client
cloudTargets []string
}
const pprofServerAddress = "127.0.0.1:8086"
@ -174,12 +176,29 @@ func New(options *types.Options) (*Runner, error) {
}()
}
if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates {
if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && (options.UpdateTemplates && !options.Cloud) {
os.Exit(0)
}
// Initialize the input source
hmapInput, err := hybrid.New(options)
hmapInput, err := hybrid.New(&hybrid.Options{
Options: options,
NotFoundCallback: func(target string) bool {
parsed, parseErr := strconv.ParseInt(target, 10, 64)
if parseErr != nil {
if err := runner.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Contents: target, Type: "targets"}); err == nil {
runner.cloudTargets = append(runner.cloudTargets, target)
return true
}
return false
}
if exists, err := runner.cloudClient.ExistsTarget(parsed); err == nil {
runner.cloudTargets = append(runner.cloudTargets, exists.Reference)
return true
}
return false
},
})
if err != nil {
return nil, errors.Wrap(err, "could not create input provider")
}
@ -401,6 +420,26 @@ func (r *Runner) RunEnumeration() error {
if err != nil {
return errors.Wrap(err, "could not load templates from config")
}
var cloudTemplates []string
if r.options.Cloud {
// hook template loading
store.NotFoundCallback = func(template string) bool {
parsed, parseErr := strconv.ParseInt(template, 10, 64)
if parseErr != nil {
if err := r.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Type: "templates", Contents: template}); err == nil {
cloudTemplates = append(cloudTemplates, template)
return true
}
return false
}
if exists, err := r.cloudClient.ExistsTemplate(parsed); err == nil {
cloudTemplates = append(cloudTemplates, exists.Reference)
return true
}
return false
}
}
if r.options.Validate {
if err := store.ValidateTemplates(); err != nil {
return err
@ -443,14 +482,36 @@ func (r *Runner) RunEnumeration() error {
var results *atomic.Bool
if r.options.Cloud {
if r.options.ScanList {
err = r.getScanList()
err = r.getScanList(r.options.OutputLimit)
} else if r.options.DeleteScan != "" {
err = r.deleteScan(r.options.DeleteScan)
} else if r.options.ScanOutput != "" {
err = r.getResults(r.options.ScanOutput)
err = r.getResults(r.options.ScanOutput, r.options.OutputLimit)
} else if r.options.ListDatasources {
err = r.listDatasources()
} else if r.options.ListTargets {
err = r.listTargets()
} else if r.options.ListTemplates {
err = r.listTemplates()
} else if r.options.AddDatasource != "" {
err = r.addCloudDataSource(r.options.AddDatasource)
} else if r.options.RemoveDatasource != "" {
err = r.removeDatasource(r.options.RemoveDatasource)
} else if r.options.AddTarget != "" {
err = r.addTarget(r.options.AddTarget)
} else if r.options.AddTemplate != "" {
err = r.addTemplate(r.options.AddTemplate)
} else if r.options.GetTarget != "" {
err = r.getTarget(r.options.GetTarget)
} else if r.options.GetTemplate != "" {
err = r.getTemplate(r.options.GetTemplate)
} else if r.options.RemoveTarget != "" {
err = r.removeTarget(r.options.RemoveTarget)
} else if r.options.RemoveTemplate != "" {
err = r.removeTemplate(r.options.RemoveTemplate)
} else {
gologger.Info().Msgf("Running scan on cloud with URL %s", r.options.CloudURL)
results, err = r.runCloudEnumeration(store, r.options.NoStore)
results, err = r.runCloudEnumeration(store, cloudTemplates, r.cloudTargets, r.options.NoStore, r.options.OutputLimit)
enumeration = true
}
} else {
@ -615,9 +676,10 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
if len(store.Workflows()) > 0 {
gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows()))
}
if r.hmapInputProvider.Count() > 0 {
gologger.Info().Msgf("Targets loaded for scan: %d", r.hmapInputProvider.Count())
}
}
func (r *Runner) readNewTemplatesWithVersionFile(version string) ([]string, error) {
resp, err := http.DefaultClient.Get(fmt.Sprintf("https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/%s/.new-additions", version))

2
v2/internal/runner/update.go
View File

@ -86,7 +86,7 @@ func (r *Runner) updateTemplates() error { // TODO this method does more than ju
}
// if disable update check flag is passed and no update template flag is passed
if r.options.NoUpdateTemplates && !r.options.UpdateTemplates {
if (r.options.NoUpdateTemplates && !r.options.UpdateTemplates) || r.options.Cloud {
return nil
}

2
v2/pkg/catalog/catalogue.go
View File

@ -12,7 +12,7 @@ type Catalog interface {
// or folders provided as in.
GetTemplatePath(target string) ([]string, error)
// GetTemplatesPath returns a list of absolute paths for the provided template list.
GetTemplatesPath(definitions []string) []string
GetTemplatesPath(definitions []string) ([]string, map[string]error)
// ResolvePath resolves the path to an absolute one in various ways.
//
// It checks if the filename is an absolute path, looks in the current directory

8
v2/pkg/catalog/disk/find.go
View File

@ -7,14 +7,14 @@ import (
"strings"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
)
// GetTemplatesPath returns a list of absolute paths for the provided template list.
func (c *DiskCatalog) GetTemplatesPath(definitions []string) []string {
func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[string]error) {
// keeps track of processed dirs and files
processed := make(map[string]bool)
allTemplates := []string{}
erred := make(map[string]error)
for _, t := range definitions {
if strings.HasPrefix(t, "http") && (strings.HasSuffix(t, ".yaml") || strings.HasSuffix(t, ".yml")) {
@ -25,7 +25,7 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) []string {
} else {
paths, err := c.GetTemplatePath(t)
if err != nil {
gologger.Error().Msgf("Could not find template '%s': %s\n", t, err)
erred[t] = err
}
for _, path := range paths {
if _, ok := processed[path]; !ok {
@ -35,7 +35,7 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) []string {
}
}
}
return allTemplates
return allTemplates, erred
}
// GetTemplatePath parses the specified input template path and returns a compiled

18
v2/pkg/catalog/disk/path.go
View File

@ -6,8 +6,6 @@ import (
"path/filepath"
"github.com/pkg/errors"
folderutil "github.com/projectdiscovery/utils/folder"
)
// ResolvePath resolves the path to an absolute one in various ways.
@ -49,20 +47,8 @@ var errNoValidCombination = errors.New("no valid combination found")
// tryResolve attempts to load locate the target by iterating across all the folders tree
func (c *DiskCatalog) tryResolve(fullPath string) (string, error) {
dir, filename := filepath.Split(fullPath)
pathInfo, err := folderutil.NewPathInfo(dir)
if err != nil {
return "", err
if _, err := os.Stat(fullPath); !os.IsNotExist(err) {
return fullPath, nil
}
pathInfoItems, err := pathInfo.MeshWith(filename)
if err != nil {
return "", err
}
for _, pathInfoItem := range pathInfoItems {
if _, err := os.Stat(pathInfoItem); !os.IsNotExist(err) {
return pathInfoItem, nil
}
}
return "", errNoValidCombination
}

5
v2/pkg/catalog/loader/filter/path_filter.go
View File

@ -18,12 +18,13 @@ type PathFilterConfig struct {
// NewPathFilter creates a new path filter from provided config
func NewPathFilter(config *PathFilterConfig, catalogClient catalog.Catalog) *PathFilter {
paths, _ := catalogClient.GetTemplatesPath(config.ExcludedTemplates)
filter := &PathFilter{
excludedTemplates: catalogClient.GetTemplatesPath(config.ExcludedTemplates),
excludedTemplates: paths,
alwaysIncludedTemplatesMap: make(map[string]struct{}),
}
alwaysIncludeTemplates := catalogClient.GetTemplatesPath(config.IncludedTemplates)
alwaysIncludeTemplates, _ := catalogClient.GetTemplatesPath(config.IncludedTemplates)
for _, tpl := range alwaysIncludeTemplates {
filter.alwaysIncludedTemplatesMap[tpl] = struct{}{}
}

27
v2/pkg/catalog/loader/loader.go
View File

@ -58,6 +58,10 @@ type Store struct {
workflows []*templates.Template
preprocessor templates.Preprocessor
// NotFoundCallback is called for each not found template
// This overrides error handling for not found templatesss
NotFoundCallback func(template string) bool
}
// NewConfig returns a new loader config
@ -174,8 +178,10 @@ func init() {
// ValidateTemplates takes a list of templates and validates them
// erroring out on discovering any faulty templates.
func (store *Store) ValidateTemplates() error {
templatePaths := store.config.Catalog.GetTemplatesPath(store.finalTemplates)
workflowPaths := store.config.Catalog.GetTemplatesPath(store.finalWorkflows)
templatePaths, errs := store.config.Catalog.GetTemplatesPath(store.finalTemplates)
store.logErroredTemplates(errs)
workflowPaths, errs := store.config.Catalog.GetTemplatesPath(store.finalWorkflows)
store.logErroredTemplates(errs)
filteredTemplatePaths := store.pathFilter.Match(templatePaths)
filteredWorkflowPaths := store.pathFilter.Match(workflowPaths)
@ -263,7 +269,8 @@ func isParsingError(message string, template string, err error) bool {
// LoadTemplates takes a list of templates and returns paths for them
func (store *Store) LoadTemplates(templatesList []string) []*templates.Template {
includedTemplates := store.config.Catalog.GetTemplatesPath(templatesList)
includedTemplates, errs := store.config.Catalog.GetTemplatesPath(templatesList)
store.logErroredTemplates(errs)
templatePathMap := store.pathFilter.Match(includedTemplates)
loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
@ -295,7 +302,8 @@ func (store *Store) LoadTemplates(templatesList []string) []*templates.Template
// LoadWorkflows takes a list of workflows and returns paths for them
func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
includedWorkflows := store.config.Catalog.GetTemplatesPath(workflowsList)
includedWorkflows, errs := store.config.Catalog.GetTemplatesPath(workflowsList)
store.logErroredTemplates(errs)
workflowPathMap := store.pathFilter.Match(includedWorkflows)
loadedWorkflows := make([]*templates.Template, 0, len(workflowPathMap))
@ -319,7 +327,8 @@ func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template
// LoadTemplatesWithTags takes a list of templates and extra tags
// returning templates that match.
func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templates.Template {
includedTemplates := store.config.Catalog.GetTemplatesPath(templatesList)
includedTemplates, errs := store.config.Catalog.GetTemplatesPath(templatesList)
store.logErroredTemplates(errs)
templatePathMap := store.pathFilter.Match(includedTemplates)
loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
@ -382,3 +391,11 @@ func workflowContainsProtocol(workflow []*workflows.WorkflowTemplate) bool {
}
return false
}
func (s *Store) logErroredTemplates(erred map[string]error) {
for template, err := range erred {
if s.NotFoundCallback == nil || !s.NotFoundCallback(template) {
gologger.Error().Msgf("Could not find template '%s': %s", template, err)
}
}
}

27
v2/pkg/core/inputs/hybrid/hmap.go
View File

@ -41,9 +41,20 @@ type Input struct {
sync.Once
}
// Options is a wrapper around types.Options structure
type Options struct {
// Options contains options for hmap provider
Options *types.Options
// NotFoundCallback is called for each not found target
// This overrides error handling for not found target
NotFoundCallback func(template string) bool
}
// New creates a new hmap backed nuclei Input Provider
// and initializes it based on the passed options Model.
func New(options *types.Options) (*Input, error) {
func New(opts *Options) (*Input, error) {
options := opts.Options
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
if err != nil {
return nil, errors.Wrap(err, "could not create temporary input file")
@ -71,7 +82,7 @@ func New(options *types.Options) (*Input, error) {
}
input.hostMapStream = fkv
}
if initErr := input.initializeInputSources(options); initErr != nil {
if initErr := input.initializeInputSources(opts); initErr != nil {
return nil, initErr
}
if input.dupeCount > 0 {
@ -89,7 +100,9 @@ func (i *Input) Close() {
}
// initializeInputSources initializes the input sources for hmap input
func (i *Input) initializeInputSources(options *types.Options) error {
func (i *Input) initializeInputSources(opts *Options) error {
options := opts.Options
// Handle targets flags
for _, target := range options.Targets {
switch {
@ -111,11 +124,15 @@ func (i *Input) initializeInputSources(options *types.Options) error {
if options.TargetsFilePath != "" {
input, inputErr := os.Open(options.TargetsFilePath)
if inputErr != nil {
// Handle cloud based input here.
if opts.NotFoundCallback == nil || !opts.NotFoundCallback(options.TargetsFilePath) {
return errors.Wrap(inputErr, "could not open targets file")
}
defer input.Close()
}
if input != nil {
i.scanInputFromReader(input)
input.Close()
}
}
if options.Uncover && options.UncoverQuery != nil {
gologger.Info().Msgf("Running uncover query against: %s", strings.Join(options.UncoverEngine, ","))

4
v2/pkg/external/customtemplates/github.go vendored
View File

@ -30,7 +30,7 @@ func (customTemplate *customTemplateGithubRepo) Download(location string, ctx co
if !fileutil.FolderExists(clonePath) {
err := customTemplate.cloneRepo(clonePath, customTemplate.githubToken)
if err != nil {
gologger.Info().Msgf("%s", err)
gologger.Error().Msgf("%s", err)
} else {
gologger.Info().Msgf("Repo %s/%s cloned successfully at %s", customTemplate.owner, customTemplate.reponame, clonePath)
}
@ -49,7 +49,7 @@ func (customTemplate *customTemplateGithubRepo) Update(location string, ctx cont
}
err := customTemplate.pullChanges(clonePath, customTemplate.githubToken)
if err != nil {
gologger.Info().Msgf("%s", err)
gologger.Error().Msgf("%s", err)
} else {
gologger.Info().Msgf("Repo %s/%s successfully pulled the changes.\n", customTemplate.owner, customTemplate.reponame)
}

7
v2/pkg/external/customtemplates/templates_provider.go vendored
View File

@ -21,18 +21,21 @@ type Provider interface {
// parseCustomTemplates function reads the options.GithubTemplateRepo list,
// Checks the given repos are valid or not and stores them into runner.CustomTemplates
func ParseCustomTemplates(options *types.Options) []Provider {
if options.Cloud {
return nil
}
var customTemplates []Provider
gitHubClient := getGHClientIncognito()
for _, repoName := range options.GithubTemplateRepo {
owner, repo, err := getOwnerAndRepo(repoName)
if err != nil {
gologger.Info().Msgf("%s", err)
gologger.Error().Msgf("%s", err)
continue
}
githubRepo, err := getGithubRepo(gitHubClient, owner, repo, options.GithubToken)
if err != nil {
gologger.Info().Msgf("%s", err)
gologger.Error().Msgf("%s", err)
continue
}
customTemplateRepo := &customTemplateGithubRepo{

8
v2/pkg/model/types/severity/severities.go
View File

@ -68,6 +68,14 @@ func (severities Severities) String() string {
return strings.Join(stringSeverities, ", ")
}
func (severities Severities) MarshalJSON() ([]byte, error) {
var stringSeverities = make([]string, 0, len(severities))
for _, severity := range severities {
stringSeverities = append(stringSeverities, severity.String())
}
return json.Marshal(stringSeverities)
}
func setSeverity(severities *Severities, value string) error {
computedSeverity, err := toSeverity(value)
if err != nil {

10
v2/pkg/model/types/severity/severity_test.go
View File

@ -6,6 +6,7 @@ import (
"gopkg.in/yaml.v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestYamlUnmarshal(t *testing.T) {
@ -65,3 +66,12 @@ func unmarshal(value string, unmarshaller func(data []byte, v interface{}) error
func createYAML(value string) string {
return "severity: " + value + "\n"
}
func TestMarshalJSON(t *testing.T) {
unmarshalled := Severities{Low, Medium}
data, err := unmarshalled.MarshalJSON()
if err != nil {
panic(err)
}
require.Equal(t, "[\"low\",\"medium\"]", string(data), "could not marshal json")
}

10
v2/pkg/parsers/workflow_loader.go
View File

@ -40,7 +40,10 @@ func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error)
}
func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string {
includedTemplates := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory})
includedTemplates, errs := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory})
for template, err := range errs {
gologger.Error().Msgf("Could not find template '%s': %s", template, err)
}
templatePathMap := w.pathFilter.Match(includedTemplates)
loadedTemplates := make([]string, 0, len(templatePathMap))
@ -56,7 +59,10 @@ func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string
}
func (w *workflowLoader) GetTemplatePaths(templatesList []string, noValidate bool) []string {
includedTemplates := w.options.Catalog.GetTemplatesPath(templatesList)
includedTemplates, errs := w.options.Catalog.GetTemplatesPath(templatesList)
for template, err := range errs {
gologger.Error().Msgf("Could not find template '%s': %s", template, err)
}
templatesPathMap := w.pathFilter.Match(includedTemplates)
loadedTemplates := make([]string, 0, len(templatesPathMap))

8
v2/pkg/templates/types/types.go
View File

@ -154,6 +154,14 @@ func (protocolTypes *ProtocolTypes) UnmarshalYAML(unmarshal func(interface{}) er
return nil
}
func (protocolTypes ProtocolTypes) MarshalJSON() ([]byte, error) {
var stringProtocols = make([]string, 0, len(protocolTypes))
for _, protocol := range protocolTypes {
stringProtocols = append(stringProtocols, protocol.String())
}
return json.Marshal(stringProtocols)
}
func (protocolTypes ProtocolTypes) String() string {
var stringTypes []string
for _, t := range protocolTypes {

31
v2/pkg/types/types.go
View File

@ -97,10 +97,34 @@ type Options struct {
CloudAPIKey string
// Scanlist feature to get all the scan ids for a user
ScanList bool
// ListDatasources enables listing of datasources for user
ListDatasources bool
// ListTargets enables listing of targets for user
ListTargets bool
// ListTemplates enables listing of templates for user
ListTemplates bool
// Limit the number of items at a time
OutputLimit int
// Nostore
NoStore bool
// Delete scan
DeleteScan string
// AddDatasource adds a datasource to cloud storage
AddDatasource string
// RemoveDatasource deletes a datasource from cloud storage
RemoveDatasource string
// AddTemplate adds a list of templates to custom datasource
AddTemplate string
// AddTarget adds a list of targets to custom datasource
AddTarget string
// GetTemplate gets a template by id
GetTemplate string
// GetTarget gets a target by id
GetTarget string
// RemoveTemplate removes a list of templates
RemoveTemplate string
// RemoveTarget removes a list of targets
RemoveTarget string
// Get issues for a scan
ScanOutput string
// ResolversFile is a file containing resolvers for nuclei.
@ -157,6 +181,8 @@ type Options struct {
Headless bool
// ShowBrowser specifies whether the show the browser in headless mode
ShowBrowser bool
// NoTables disables pretty printing of cloud results in tables
NoTables bool
// DisableClustering disables clustering of templates
DisableClustering bool
// UseInstalledChrome skips chrome install and use local instance
@ -352,3 +378,8 @@ func DefaultOptions() *Options {
MaxHostError: 30,
}
}
// HasCloudOptions returns true if cloud options have been specified
func (options *Options) HasCloudOptions() bool {
return options.ScanList || options.DeleteScan != "" || options.ScanOutput != "" || options.ListDatasources || options.ListTargets || options.ListTemplates || options.RemoveDatasource != "" || options.AddTarget != "" || options.AddTemplate != "" || options.RemoveTarget != "" || options.RemoveTemplate != "" || options.GetTarget != "" || options.GetTemplate != ""
}