From 96646c8f53d908bc2c009ae658ff4cec7cd85d55 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Wed, 21 Dec 2022 22:48:43 +0530 Subject: [PATCH] cloud templates targets sync (#2959) * Add s3 bucket template provider - Refactor the custom github template code - add interface for template provider * Validate if aws creds are passed if bucket flag - refactor s3 provider struct to take client - add function which returns the aws s3 client - update error messages * Add aws s3 bucket flags documentation in README.md - Rename the github_test.go to customTemplate_test.go * go mod update * Move template provider code to pkg/external/customtemplates dir * Added initial data_source sync to cloud * Misc * Add pagination to scan output and scan list (#2858) * Add pagination to scan output and scan list * Use time based parameters instead of page numbers * Fix linting errors * Do not check limits at client, check at server * Remove unused constant * Misc update * Removed unnecessary flags * Misc * Misc * Misc endpoint additions * Added more routes * Typo fix * Misc fixes * Misc * Misc fixes to cloud target logic + use int for IDs * Misc * Misc fixes * Misc * Misc fixes * readme update * Add JSON output support for list-scan option (#2876) * Add JSON output support for list-scan option * Fix typo in cloud JSON output description * Following changes - Update status(finished, running) to be lower-case by default - Convert status to upper-case in DisplayScanList() * Update status to be lower-case by default * Remove additional json flag, instead use existing * Merge conflict * Accomodate comment changes and restructure code Co-authored-by: Jaideep K * Use integer IDs for scan tasks * Added get-templates-targets endpoint + JSON + validation * Added target count list * misc option / description updates * Added changes as per code review * duplicate options + typo updates * Added tablewriter for tabular data writing by default * Fixed list scan endpoint * Review changes * workflow fix * Added cloud tags etc based filtering (#3070) * Added omitempty for filtering request * go mod tidy * misc format update Co-authored-by: shubhamrasal Co-authored-by: Ice3man Co-authored-by: Jaideep Khandelwal Co-authored-by: Siddharth Shashikar <60960197+shashikarsiddharth@users.noreply.github.com> Co-authored-by: Jaideep K --- .github/workflows/publish-docs.yaml | 9 +- README.md | 8 +- v2/cmd/nuclei/main.go | 23 +- v2/go.mod | 18 +- v2/go.sum | 4 +- v2/internal/runner/cloud.go | 319 ++++++++++++ v2/internal/runner/enumerate.go | 81 ++- v2/internal/runner/nucleicloud/cloud.go | 465 +++++++++++++++--- v2/internal/runner/nucleicloud/types.go | 115 ++++- v2/internal/runner/nucleicloud/utils.go | 29 ++ v2/internal/runner/options.go | 73 ++- v2/internal/runner/runner.go | 76 ++- v2/internal/runner/update.go | 2 +- v2/pkg/catalog/catalogue.go | 2 +- v2/pkg/catalog/disk/find.go | 8 +- v2/pkg/catalog/disk/path.go | 18 +- v2/pkg/catalog/loader/filter/path_filter.go | 5 +- v2/pkg/catalog/loader/loader.go | 27 +- v2/pkg/core/inputs/hybrid/hmap.go | 31 +- v2/pkg/external/customtemplates/github.go | 4 +- .../customtemplates/templates_provider.go | 7 +- v2/pkg/model/types/severity/severities.go | 8 + v2/pkg/model/types/severity/severity_test.go | 10 + v2/pkg/parsers/workflow_loader.go | 10 +- v2/pkg/templates/types/types.go | 8 + v2/pkg/types/types.go | 31 ++ 26 files changed, 1196 insertions(+), 195 deletions(-) create mode 100644 v2/internal/runner/cloud.go diff --git a/.github/workflows/publish-docs.yaml b/.github/workflows/publish-docs.yaml index a36c0fbf..0b4e2a76 100644 --- a/.github/workflows/publish-docs.yaml +++ b/.github/workflows/publish-docs.yaml @@ -12,12 +12,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 - with: - fetch-depth: 0 - name: "Set up Go" uses: actions/setup-go@v3 - with: + with: go-version: 1.18 - name: Generate YAML Syntax Documentation @@ -28,7 +26,7 @@ jobs: go install github.com/projectdiscovery/yamldoc-go/cmd/docgen/dstdocgen@main fi go generate pkg/templates/templates.go - go build -o "cmd/docgen/docgen" cmd/docgen/docgen.go + go build -o "cmd/docgen/docgen" cmd/docgen/docgen.go ./cmd/docgen/docgen ../SYNTAX-REFERENCE.md ../nuclei-jsonschema.json git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT working-directory: v2 @@ -38,6 +36,7 @@ jobs: run: | git config --local user.email "action@github.com" git config --local user.name "GitHub Action" + git pull git add SYNTAX-REFERENCE.md nuclei-jsonschema.json git commit -m "Auto Generate Syntax Docs + JSONSchema [$(date)] :robot:" -a @@ -46,4 +45,4 @@ jobs: uses: ad-m/github-push-action@master with: github_token: ${{ secrets.GITHUB_TOKEN }} - branch: ${{ github.ref }} + branch: ${{ github.head_ref }} diff --git a/README.md b/README.md index ed58fc23..ef162695 100644 --- a/README.md +++ b/README.md @@ -239,10 +239,10 @@ DEBUG: -hc, -health-check run diagnostic check up UPDATE: - -un, -update update nuclei engine to the latest released version - -ut, -update-templates update nuclei-templates to latest released version - -ud, -update-template-dir string custom directory to install / update nuclei-templates - -duc, -disable-update-check disable automatic nuclei/templates update check + -un, -update update nuclei engine to the latest released version + -ut, -update-templates update nuclei-templates to latest released version + -ud, -update-template-dir string custom directory to install / update nuclei-templates + -duc, -disable-update-check disable automatic nuclei/templates update check STATISTICS: -stats display statistics about the running scan diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 94cfa113..04f84e78 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -295,12 +295,23 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.CreateGroup("cloud", "Cloud", flagSet.BoolVar(&options.Cloud, "cloud", false, "run scan on nuclei cloud"), - flagSet.StringVarEnv(&options.CloudURL, "cloud-server", "cs", "https://cloud-dev.nuclei.sh", "NUCLEI_CLOUD_SERVER", "nuclei cloud server to use (NUCLEI_CLOUD_SERVER)"), - flagSet.StringVarEnv(&options.CloudAPIKey, "cloud-api-key", "ak", "", "NUCLEI_CLOUD_APIKEY", "api-key for the nuclei cloud server (NUCLEI_CLOUD_APIKEY)"), - flagSet.BoolVarP(&options.ScanList, "list-scan", "ls", false, "list previous cloud scans"), - flagSet.BoolVarP(&options.NoStore, "no-store", "ns", false, "disable scan/output storage on cloud"), - flagSet.StringVarP(&options.DeleteScan, "delete-scan", "ds", "", "delete scan/output on cloud by scan id"), - flagSet.StringVarP(&options.ScanOutput, "scan-output", "so", "", "display scan output by scan id"), + flagSet.StringVarP(&options.AddDatasource, "add-datasource", "ads", "", "add specified data source (s3,github)"), + flagSet.StringVarP(&options.AddTarget, "add-target", "atr", "", "add target(s) to cloud"), + flagSet.StringVarP(&options.AddTemplate, "add-template", "atm", "", "add template(s) to cloud"), + flagSet.BoolVarP(&options.ScanList, "list-scan", "lsn", false, "list previous cloud scans"), + flagSet.BoolVarP(&options.ListTargets, "list-target", "ltr", false, "list cloud target by id"), + flagSet.BoolVarP(&options.ListTemplates, "list-template", "ltm", false, "list cloud template by id"), + flagSet.BoolVarP(&options.ListDatasources, "list-datasource", "lds", false, "list cloud datasource by id"), + flagSet.StringVarP(&options.DeleteScan, "delete-scan", "dsn", "", "delete cloud scan by id"), + flagSet.StringVarP(&options.RemoveTarget, "delete-target", "dtr", "", "delete target(s) from cloud"), + flagSet.StringVarP(&options.RemoveTemplate, "delete-template", "dtm", "", "delete template(s) from cloud"), + flagSet.StringVarP(&options.RemoveDatasource, "delete-datasource", "dds", "", "delete specified data source"), + flagSet.StringVarP(&options.GetTarget, "get-target", "gtr", "", "get target content by id"), + flagSet.StringVarP(&options.GetTemplate, "get-template", "gtm", "", "get template content by id"), + flagSet.BoolVarP(&options.NoStore, "no-store", "nos", false, "disable scan/output storage on cloud"), + flagSet.StringVarP(&options.ScanOutput, "scan-output", "sno", "", "display scan output by scan id"), + flagSet.BoolVar(&options.NoTables, "no-tables", false, "do not display pretty-printed tables"), + flagSet.IntVar(&options.OutputLimit, "limit", 100, "limit the number of output to display"), ) _ = flagSet.Parse() diff --git a/v2/go.mod b/v2/go.mod index 5c99c6fa..94e12723 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -30,7 +30,7 @@ require ( github.com/projectdiscovery/interactsh v1.0.6-0.20220827132222-460cc6270053 github.com/projectdiscovery/rawhttp v0.1.4 github.com/projectdiscovery/retryabledns v1.0.17 - github.com/projectdiscovery/retryablehttp-go v1.0.5-0.20221202084821-c1a692a64751 + github.com/projectdiscovery/retryablehttp-go v1.0.6-0.20221206071935-7924d7d34953 github.com/projectdiscovery/stringsutil v0.0.2 github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 github.com/remeh/sizedwaitgroup v1.0.0 @@ -87,7 +87,14 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.3.1 // indirect + github.com/bits-and-blooms/bloom/v3 v3.3.1 // indirect github.com/cloudflare/circl v1.1.0 // indirect github.com/dlclark/regexp2 v1.4.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect @@ -111,6 +118,7 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/tinyqueue v0.1.1 // indirect + github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect ) @@ -126,8 +134,6 @@ require ( github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.2.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/bits-and-blooms/bitset v1.3.1 // indirect - github.com/bits-and-blooms/bloom/v3 v3.3.1 // indirect github.com/c4milo/unpackit v0.1.0 // indirect github.com/caddyserver/certmagic v0.16.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect @@ -218,16 +224,11 @@ require ( github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect github.com/alecthomas/chroma v0.10.0 - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.20 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.27 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.20 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.26 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.9 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.17.6 // indirect @@ -250,6 +251,5 @@ require ( github.com/src-d/gcfg v1.4.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect - github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 6b52d2b5..39726757 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -637,8 +637,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.17 h1:XKzI26UKYt2g7YLJ/EcyYmM04sfD1vurETecPEpeA1w= github.com/projectdiscovery/retryabledns v1.0.17/go.mod h1:Dyhq/f0sGmXueso0+Ah3LbJfsX4PXpBrpfiyjZZ8SDk= -github.com/projectdiscovery/retryablehttp-go v1.0.5-0.20221202084821-c1a692a64751 h1:QEmZ0E6GDzlTbVE6ty7fCuKR7muWrLqMfQ07VTu6Bd0= -github.com/projectdiscovery/retryablehttp-go v1.0.5-0.20221202084821-c1a692a64751/go.mod h1:B/xfvUmiJBeq+1kT7AMYL6B/IuPgbyKB7QCKPSfMByc= +github.com/projectdiscovery/retryablehttp-go v1.0.6-0.20221206071935-7924d7d34953 h1:8voTfrLQKt/nElO7zx6yoaFZDCeDq2xFHkWEl3fXBBI= +github.com/projectdiscovery/retryablehttp-go v1.0.6-0.20221206071935-7924d7d34953/go.mod h1:Z9FoXiCxITq+Wt6VzqJAu8xA/xDTVrgBFaDk+Xsfgw4= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/sliceutil v0.0.1 h1:YoCqCMcdwz+gqNfW5hFY8UvNHoA6SfyBSNkVahatleg= diff --git a/v2/internal/runner/cloud.go b/v2/internal/runner/cloud.go new file mode 100644 index 00000000..9cfe2601 --- /dev/null +++ b/v2/internal/runner/cloud.go @@ -0,0 +1,319 @@ +package runner + +import ( + "io" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" + + jsoniter "github.com/json-iterator/go" + "github.com/olekukonko/tablewriter" + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/internal/runner/nucleicloud" + "github.com/projectdiscovery/nuclei/v2/pkg/output" +) + +// Get all the scan lists for a user/apikey. +func (r *Runner) getScanList(limit int) error { + lastTime := "2099-01-02 15:04:05 +0000 UTC" + header := []string{"ID", "Timestamp", "Targets", "Templates", "Matched", "Duration", "Status"} + + var ( + values [][]string + count int + ) + for { + items, err := r.cloudClient.GetScans(limit, lastTime) + if err != nil { + return err + } + if len(items) == 0 { + break + } + for _, v := range items { + count++ + lastTime = v.CreatedAt.String() + res := nucleicloud.PrepareScanListOutput(v) + if r.options.JSON { + _ = jsoniter.NewEncoder(os.Stdout).Encode(res) + } else if !r.options.NoTables { + values = append(values, []string{strconv.FormatInt(res.ScanID, 10), res.Timestamp, strconv.Itoa(res.Target), strconv.Itoa(res.Template), strconv.Itoa(res.ScanResult), res.ScanTime, res.ScanStatus}) + } else { + gologger.Silent().Msgf("%d. [%s] [TARGETS: %d] [TEMPLATES: %d] [MATCHED: %d] [DURATION: %s] [STATUS: %s]\n", res.ScanID, res.Timestamp, res.Target, res.Template, res.ScanResult, res.ScanTime, strings.ToUpper(res.ScanStatus)) + + } + } + } + if count == 0 { + return errors.New("no scan found") + } + if !r.options.NoTables { + r.prettyPrintTable(header, values) + } + return nil +} + +func (r *Runner) listDatasources() error { + datasources, err := r.cloudClient.ListDatasources() + if err != nil { + return err + } + if len(datasources) == 0 { + return errors.New("no cloud datasource found") + } + + header := []string{"ID", "UpdatedAt", "Type", "Repo", "Path"} + var values [][]string + for _, source := range datasources { + if r.options.JSON { + _ = jsoniter.NewEncoder(os.Stdout).Encode(source) + } else if !r.options.NoTables { + values = append(values, []string{strconv.FormatInt(source.ID, 10), source.Updatedat.Format(nucleicloud.DDMMYYYYhhmmss), source.Type, source.Repo, source.Path}) + } else { + gologger.Silent().Msgf("%d. [%s] [%s] [%s] %s", source.ID, source.Updatedat.Format(nucleicloud.DDMMYYYYhhmmss), source.Type, source.Repo, source.Path) + } + } + if !r.options.NoTables { + r.prettyPrintTable(header, values) + } + return nil +} + +func (r *Runner) listTargets() error { + items, err := r.cloudClient.ListTargets("") + if err != nil { + return err + } + if len(items) == 0 { + return errors.New("no target found") + } + + header := []string{"ID", "Reference", "Count"} + var values [][]string + for _, source := range items { + if r.options.JSON { + _ = jsoniter.NewEncoder(os.Stdout).Encode(source) + } else if !r.options.NoTables { + values = append(values, []string{strconv.FormatInt(source.ID, 10), source.Reference, strconv.FormatInt(source.Count, 10)}) + } else { + gologger.Silent().Msgf("%d. %s (%d)", source.ID, source.Reference, source.Count) + } + } + if !r.options.NoTables { + r.prettyPrintTable(header, values) + } + return nil +} + +func (r *Runner) listTemplates() error { + items, err := r.cloudClient.ListTemplates("") + if err != nil { + return err + } + if len(items) == 0 { + return errors.New("no template found") + } + + header := []string{"ID", "Reference"} + var values [][]string + for _, source := range items { + if r.options.JSON { + _ = jsoniter.NewEncoder(os.Stdout).Encode(source) + } else if !r.options.NoTables { + values = append(values, []string{strconv.FormatInt(source.ID, 10), source.Reference}) + } else { + gologger.Silent().Msgf("%d. %s", source.ID, source.Reference) + } + } + if !r.options.NoTables { + r.prettyPrintTable(header, values) + } + return nil +} + +func (r *Runner) prettyPrintTable(header []string, values [][]string) { + writer := tablewriter.NewWriter(os.Stdout) + writer.SetHeader(header) + writer.AppendBulk(values) + writer.Render() +} + +func (r *Runner) deleteScan(id string) error { + ID, _ := strconv.ParseInt(id, 10, 64) + deleted, err := r.cloudClient.DeleteScan(ID) + if !deleted.OK { + gologger.Error().Msgf("Error in deleting the scan %s.", id) + } else { + gologger.Info().Msgf("Scan deleted %s.", id) + } + return err +} + +func (r *Runner) getResults(id string, limit int) error { + ID, _ := strconv.ParseInt(id, 10, 64) + err := r.cloudClient.GetResults(ID, false, limit, func(re *output.ResultEvent) { + if outputErr := r.output.Write(re); outputErr != nil { + gologger.Warning().Msgf("Could not write output: %s", outputErr) + } + }) + return err +} + +func (r *Runner) getTarget(id string) error { + ID, _ := strconv.ParseInt(id, 10, 64) + reader, err := r.cloudClient.GetTarget(ID) + if err != nil { + return errors.Wrap(err, "could not get target") + } + defer reader.Close() + + _, _ = io.Copy(os.Stdout, reader) + return nil +} + +func (r *Runner) getTemplate(id string) error { + ID, _ := strconv.ParseInt(id, 10, 64) + reader, err := r.cloudClient.GetTemplate(ID) + if err != nil { + return errors.Wrap(err, "could not get template") + } + defer reader.Close() + + _, _ = io.Copy(os.Stdout, reader) + return nil +} + +func (r *Runner) removeDatasource(datasource string) error { + var source string + ID, parseErr := strconv.ParseInt(datasource, 10, 64) + if parseErr != nil { + source = datasource + } + + err := r.cloudClient.RemoveDatasource(ID, source) + if err == nil { + gologger.Info().Msgf("Datasource deleted %s", datasource) + } + return err +} + +func (r *Runner) addTemplate(location string) error { + walkErr := filepath.WalkDir(location, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".yaml") { + return nil + } + base := filepath.Base(path) + reference, templateErr := r.cloudClient.AddTemplate(base, path) + if templateErr != nil { + gologger.Error().Msgf("Could not upload %s: %s", path, templateErr) + } else if reference != "" { + gologger.Info().Msgf("Uploaded template %s: %s", base, reference) + } + return nil + }) + return walkErr +} + +func (r *Runner) addTarget(location string) error { + walkErr := filepath.WalkDir(location, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".txt") { + return nil + } + base := filepath.Base(location) + reference, targetErr := r.cloudClient.AddTarget(base, location) + if targetErr != nil { + gologger.Error().Msgf("Could not upload %s: %s", location, targetErr) + } else if reference != "" { + gologger.Info().Msgf("Uploaded target %s: %s", base, reference) + } + return nil + }) + return walkErr +} + +func (r *Runner) removeTarget(item string) error { + response, err := r.cloudClient.ListTargets(item) + if err != nil { + return errors.Wrap(err, "could not list targets") + } + for _, item := range response { + if err := r.cloudClient.RemoveTarget(item.ID); err != nil { + gologger.Error().Msgf("Error in deleting target %s: %s", item.Reference, err) + } else { + gologger.Info().Msgf("Target deleted %s", item.Reference) + } + } + return nil +} + +func (r *Runner) removeTemplate(item string) error { + response, err := r.cloudClient.ListTemplates(item) + if err != nil { + return errors.Wrap(err, "could not list templates") + } + for _, item := range response { + if err := r.cloudClient.RemoveTemplate(item.ID); err != nil { + gologger.Error().Msgf("Error in deleting template %s: %s", item.Reference, err) + } else { + gologger.Info().Msgf("Template deleted %s", item.Reference) + } + } + return nil +} + +// initializeCloudDataSources initializes cloud data sources +func (r *Runner) addCloudDataSource(source string) error { + switch source { + case "s3": + token := strings.Join([]string{r.options.AwsAccessKey, r.options.AwsSecretKey, r.options.AwsRegion}, ":") + if _, err := r.processDataSourceItem(r.options.AwsBucketName, token, "s3"); err != nil { + return err + } + case "github": + for _, repo := range r.options.GithubTemplateRepo { + if _, err := r.processDataSourceItem(repo, r.options.GithubToken, "github"); err != nil { + return err + } + } + } + return nil +} + +func (r *Runner) processDataSourceItem(repo, token, Type string) (int64, error) { + ID, err := r.cloudClient.StatusDataSource(nucleicloud.StatusDataSourceRequest{Repo: repo, Token: token}) + if err != nil { + if !strings.Contains(err.Error(), "no rows in result set") { + return 0, errors.Wrap(err, "could not get data source status") + } + + gologger.Info().Msgf("Adding new data source + syncing: %s\n", repo) + resp, err := r.cloudClient.AddDataSource(nucleicloud.AddDataSourceRequest{Type: Type, Repo: repo, Token: token}) + if err != nil { + return 0, errors.Wrap(err, "could not add data source") + } + ID = resp.ID + if err = r.cloudClient.SyncDataSource(resp.ID); err != nil { + return 0, errors.Wrap(err, "could not sync data source") + } + if resp.Secret != "" { + gologger.Info().Msgf("Webhook URL for added source: %s/datasources/%s/webhook", r.options.CloudURL, resp.Hash) + gologger.Info().Msgf("Secret for webhook: %s", resp.Secret) + } + } + if r.options.UpdateTemplates { + gologger.Info().Msgf("Syncing data source: %s (%d)\n", repo, ID) + if err = r.cloudClient.SyncDataSource(ID); err != nil { + return 0, errors.Wrap(err, "could not sync data source") + } + } + return ID, nil +} diff --git a/v2/internal/runner/enumerate.go b/v2/internal/runner/enumerate.go index f6030267..d4388546 100644 --- a/v2/internal/runner/enumerate.go +++ b/v2/internal/runner/enumerate.go @@ -20,11 +20,10 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "go.uber.org/atomic" ) -const DDMMYYYYhhmmss = "2006-01-02 15:04:05" - // runStandardEnumeration runs standard enumeration func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions, store *loader.Store, engine *core.Engine) (*atomic.Bool, error) { if r.options.AutomaticScan { @@ -33,52 +32,12 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions, return r.executeTemplatesInput(store, engine) } -// Get all the scan lists for a user/apikey. -func (r *Runner) getScanList() error { - items, err := r.cloudClient.GetScans() - loc, _ := time.LoadLocation("Local") - - for _, v := range items { - status := "FINISHED" - t := v.FinishedAt - duration := t.Sub(v.CreatedAt) - if !v.Finished { - status = "RUNNING" - t = time.Now().UTC() - duration = t.Sub(v.CreatedAt).Round(60 * time.Second) - } - - val := v.CreatedAt.In(loc).Format(DDMMYYYYhhmmss) - - gologger.Silent().Msgf("%s [%s] [STATUS: %s] [MATCHED: %d] [TARGETS: %d] [TEMPLATES: %d] [DURATION: %s]\n", v.Id, val, status, v.Matches, v.Targets, v.Templates, duration) - } - return err -} - -func (r *Runner) deleteScan(id string) error { - deleted, err := r.cloudClient.DeleteScan(id) - if !deleted.OK { - gologger.Info().Msgf("Error in deleting the scan %s.", id) - } else { - gologger.Info().Msgf("Scan deleted %s.", id) - } - return err -} - -func (r *Runner) getResults(id string) error { - err := r.cloudClient.GetResults(id, func(re *output.ResultEvent) { - if outputErr := r.output.Write(re); outputErr != nil { - gologger.Warning().Msgf("Could not write output: %s", outputErr) - } - }, false) - return err -} - // runCloudEnumeration runs cloud based enumeration -func (r *Runner) runCloudEnumeration(store *loader.Store, nostore bool) (*atomic.Bool, error) { +func (r *Runner) runCloudEnumeration(store *loader.Store, cloudTemplates, cloudTargets []string, nostore bool, limit int) (*atomic.Bool, error) { + count := &atomic.Int64{} now := time.Now() defer func() { - gologger.Info().Msgf("Scan execution took %s", time.Since(now)) + gologger.Info().Msgf("Scan execution took %s and found %d results", time.Since(now), count.Load()) }() results := &atomic.Bool{} @@ -110,17 +69,24 @@ func (r *Runner) runCloudEnumeration(store *loader.Store, nostore bool) (*atomic taskID, err := r.cloudClient.AddScan(&nucleicloud.AddScanRequest{ RawTargets: targets, PublicTemplates: templates, + CloudTargets: cloudTargets, + CloudTemplates: cloudTemplates, PrivateTemplates: privateTemplates, IsTemporary: nostore, + Filtering: getCloudFilteringFromOptions(r.options), }) if err != nil { return results, err } - gologger.Info().Msgf("Created task with ID: %s", taskID) + gologger.Info().Msgf("Created task with ID: %d", taskID) + if nostore { + gologger.Info().Msgf("Cloud scan storage: disabled") + } time.Sleep(3 * time.Second) - err = r.cloudClient.GetResults(taskID, func(re *output.ResultEvent) { + err = r.cloudClient.GetResults(taskID, true, limit, func(re *output.ResultEvent) { results.CompareAndSwap(false, true) + _ = count.Inc() if outputErr := r.output.Write(re); outputErr != nil { gologger.Warning().Msgf("Could not write output: %s", err) @@ -130,7 +96,7 @@ func (r *Runner) runCloudEnumeration(store *loader.Store, nostore bool) (*atomic gologger.Warning().Msgf("Could not create issue on tracker: %s", err) } } - }, true) + }) return results, err } @@ -150,3 +116,22 @@ func gzipBase64EncodeData(data []byte) string { encoded := base64.StdEncoding.EncodeToString(buf.Bytes()) return encoded } + +func getCloudFilteringFromOptions(options *types.Options) *nucleicloud.AddScanRequestConfiguration { + return &nucleicloud.AddScanRequestConfiguration{ + Authors: options.Authors, + Tags: options.Tags, + ExcludeTags: options.ExcludeTags, + IncludeTags: options.IncludeTags, + IncludeIds: options.IncludeIds, + ExcludeIds: options.ExcludeIds, + IncludeTemplates: options.IncludeTemplates, + ExcludedTemplates: options.ExcludedTemplates, + ExcludeMatchers: options.ExcludeMatchers, + Severities: options.Severities, + ExcludeSeverities: options.ExcludeSeverities, + Protocols: options.Protocols, + ExcludeProtocols: options.ExcludeProtocols, + IncludeConditions: options.IncludeConditions, + } +} diff --git a/v2/internal/runner/nucleicloud/cloud.go b/v2/internal/runner/nucleicloud/cloud.go index 8557b686..7a6b7a03 100644 --- a/v2/internal/runner/nucleicloud/cloud.go +++ b/v2/internal/runner/nucleicloud/cloud.go @@ -3,9 +3,15 @@ package nucleicloud import ( "bytes" "context" + "encoding/json" "fmt" "io" + "mime/multipart" "net/http" + "net/url" + "os" + "path/filepath" + "strconv" "strings" "time" @@ -23,9 +29,9 @@ type Client struct { } const ( - pollInterval = 1 * time.Second - defaultBaseURL = "http://webapp.localhost" + pollInterval = 3 * time.Second resultSize = 100 + defaultBaseURL = "https://cloud-dev.nuclei.sh" ) // HTTPErrorRetryPolicy is to retry for HTTPCodes >= 500. @@ -41,7 +47,8 @@ func HTTPErrorRetryPolicy() func(ctx context.Context, resp *http.Response, err e // New returns a nuclei-cloud API client func New(baseURL, apiKey string) *Client { options := retryablehttp.DefaultOptionsSingle - options.Timeout = 15 * time.Second + options.NoAdjustTimeout = true + options.Timeout = 60 * time.Second options.CheckRetry = HTTPErrorRetryPolicy() client := retryablehttp.NewClient(options) @@ -53,57 +60,47 @@ func New(baseURL, apiKey string) *Client { } // AddScan adds a scan for templates and target to nuclei server -func (c *Client) AddScan(req *AddScanRequest) (string, error) { +func (c *Client) AddScan(req *AddScanRequest) (int64, error) { var buf bytes.Buffer if err := jsoniter.NewEncoder(&buf).Encode(req); err != nil { - return "", errors.Wrap(err, "could not json encode scan request") + return 0, errors.Wrap(err, "could not encode request") } httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/scan", c.baseURL), bytes.NewReader(buf.Bytes())) if err != nil { - return "", errors.Wrap(err, "could not make request") + return 0, errors.Wrap(err, "could not make request") } - httpReq.Header.Set("X-API-Key", c.apiKey) - resp, err := c.httpclient.Do(httpReq) + resp, err := c.sendRequest(httpReq) if err != nil { - return "", errors.Wrap(err, "could not do add scan request") + return 0, errors.Wrap(err, "could not do request") } - if resp.StatusCode != 200 { - data, _ := io.ReadAll(resp.Body) - resp.Body.Close() - return "", errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data)) - } - var data map[string]string + defer resp.Body.Close() + + var data map[string]int64 if err := jsoniter.NewDecoder(resp.Body).Decode(&data); err != nil { - resp.Body.Close() - return "", errors.Wrap(err, "could not decode resp") + return 0, errors.Wrap(err, "could not decode resp") } - resp.Body.Close() id := data["id"] return id, nil } // GetResults gets results from nuclei server for an ID // until there are no more results left to retrieve. -func (c *Client) GetResults(ID string, callback func(*output.ResultEvent), checkProgress bool) error { +func (c *Client) GetResults(ID int64, checkProgress bool, limit int, callback func(*output.ResultEvent)) error { lastID := int64(0) + for { - uri := fmt.Sprintf("%s/results?id=%s&from=%d&size=%d", c.baseURL, ID, lastID, resultSize) + uri := fmt.Sprintf("%s/results?id=%d&from=%d&size=%d", c.baseURL, ID, lastID, limit) httpReq, err := retryablehttp.NewRequest(http.MethodGet, uri, nil) if err != nil { return errors.Wrap(err, "could not make request") } - httpReq.Header.Set("X-API-Key", c.apiKey) - resp, err := c.httpclient.Do(httpReq) + resp, err := c.sendRequest(httpReq) if err != nil { - return errors.Wrap(err, "could not do ger result request") - } - if resp.StatusCode != 200 { - data, _ := io.ReadAll(resp.Body) - resp.Body.Close() - return errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data)) + return errors.Wrap(err, "could not do request") } + var items GetResultsResponse if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil { resp.Body.Close() @@ -135,61 +132,409 @@ func (c *Client) GetResults(ID string, callback func(*output.ResultEvent), check return nil } -func (c *Client) GetScans() ([]GetScanRequest, error) { +func (c *Client) GetScans(limit int, from string) ([]GetScanRequest, error) { var items []GetScanRequest - httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/scan", c.baseURL), nil) + httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/scan?from=%s&size=%d", c.baseURL, url.QueryEscape(from), limit), nil) if err != nil { return items, errors.Wrap(err, "could not make request") } - httpReq.Header.Set("X-API-Key", c.apiKey) - resp, err := c.httpclient.Do(httpReq) + resp, err := c.sendRequest(httpReq) if err != nil { - return items, errors.Wrap(err, "could not make request.") - } - if err != nil { - return items, errors.Wrap(err, "could not do get response.") - } - if resp.StatusCode != 200 { - data, _ := io.ReadAll(resp.Body) - resp.Body.Close() - return items, errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data)) + return nil, errors.Wrap(err, "could not do request") } + defer resp.Body.Close() + if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil { - resp.Body.Close() return items, errors.Wrap(err, "could not decode results") } - resp.Body.Close() - return items, nil } // Delete a scan and it's issues by the scan id. -func (c *Client) DeleteScan(id string) (DeleteScanResults, error) { +func (c *Client) DeleteScan(id int64) (DeleteScanResults, error) { deletescan := DeleteScanResults{} - httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/scan?id=%s", c.baseURL, id), nil) + httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/scan?id=%d", c.baseURL, id), nil) if err != nil { return deletescan, errors.Wrap(err, "could not make request") } - httpReq.Header.Set("X-API-Key", c.apiKey) - resp, err := c.httpclient.Do(httpReq) + resp, err := c.sendRequest(httpReq) if err != nil { - return deletescan, errors.Wrap(err, "could not make request") - } - if err != nil { - return deletescan, errors.Wrap(err, "could not do get result request") - } - if resp.StatusCode != 200 { - data, _ := io.ReadAll(resp.Body) - resp.Body.Close() - return deletescan, errors.Errorf("could not do request %d: %s", resp.StatusCode, string(data)) + return deletescan, errors.Wrap(err, "could not do request") } + defer resp.Body.Close() + if err := jsoniter.NewDecoder(resp.Body).Decode(&deletescan); err != nil { - resp.Body.Close() return deletescan, errors.Wrap(err, "could not delete scan") } - resp.Body.Close() - return deletescan, nil } + +// StatusDataSource returns the status for a data source +func (c *Client) StatusDataSource(statusRequest StatusDataSourceRequest) (int64, error) { + var buf bytes.Buffer + if err := jsoniter.NewEncoder(&buf).Encode(statusRequest); err != nil { + return 0, errors.Wrap(err, "could not encode request") + } + httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/datasources/status", c.baseURL), bytes.NewReader(buf.Bytes())) + if err != nil { + return 0, errors.Wrap(err, "could not make request") + } + + resp, err := c.sendRequest(httpReq) + if err != nil { + return 0, errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + + var data StatusDataSourceResponse + if err := jsoniter.NewDecoder(resp.Body).Decode(&data); err != nil { + return 0, errors.Wrap(err, "could not decode resp") + } + return data.ID, nil +} + +// AddDataSource adds a new data source +func (c *Client) AddDataSource(req AddDataSourceRequest) (*AddDataSourceResponse, error) { + var buf bytes.Buffer + if err := jsoniter.NewEncoder(&buf).Encode(req); err != nil { + return nil, errors.Wrap(err, "could not encode request") + } + httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/datasources", c.baseURL), bytes.NewReader(buf.Bytes())) + if err != nil { + return nil, errors.Wrap(err, "could not make request") + } + resp, err := c.sendRequest(httpReq) + if err != nil { + return nil, errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + + var data AddDataSourceResponse + if err := jsoniter.NewDecoder(resp.Body).Decode(&data); err != nil { + return nil, errors.Wrap(err, "could not decode resp") + } + return &data, nil +} + +// SyncDataSource syncs contents for a data source. The call blocks until +// update is completed. +func (c *Client) SyncDataSource(ID int64) error { + httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/datasources/%d/sync", c.baseURL, ID), nil) + if err != nil { + return errors.Wrap(err, "could not make request") + } + + resp, err := c.sendRequest(httpReq) + if err != nil { + return errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + return nil +} + +// ExistsDataSourceItem identifies whether data source item exist +func (c *Client) ExistsDataSourceItem(req ExistsDataSourceItemRequest) error { + var buf bytes.Buffer + if err := jsoniter.NewEncoder(&buf).Encode(req); err != nil { + return errors.Wrap(err, "could not encode request") + } + httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/datasources/exists", c.baseURL), bytes.NewReader(buf.Bytes())) + if err != nil { + return errors.Wrap(err, "could not make request") + } + resp, err := c.sendRequest(httpReq) + if err != nil { + return errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + return nil +} + +func (c *Client) ListDatasources() ([]GetDataSourceResponse, error) { + var items []GetDataSourceResponse + httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/datasources", c.baseURL), nil) + if err != nil { + return items, errors.Wrap(err, "could not make request") + } + + resp, err := c.sendRequest(httpReq) + if err != nil { + return nil, errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + + if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil { + return items, errors.Wrap(err, "could not decode results") + } + return items, nil +} + +func (c *Client) ListTargets(query string) ([]GetTargetResponse, error) { + var builder strings.Builder + _, _ = builder.WriteString(c.baseURL) + _, _ = builder.WriteString("/targets") + if query != "" { + _, _ = builder.WriteString("?query=") + _, _ = builder.WriteString(url.QueryEscape(query)) + } + + var items []GetTargetResponse + httpReq, err := retryablehttp.NewRequest(http.MethodGet, builder.String(), nil) + if err != nil { + return items, errors.Wrap(err, "could not make request") + } + + resp, err := c.sendRequest(httpReq) + if err != nil { + return nil, errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + + if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil { + return items, errors.Wrap(err, "could not decode results") + } + return items, nil +} + +func (c *Client) ListTemplates(query string) ([]GetTemplatesResponse, error) { + var builder strings.Builder + _, _ = builder.WriteString(c.baseURL) + _, _ = builder.WriteString("/templates") + if query != "" { + _, _ = builder.WriteString("?query=") + _, _ = builder.WriteString(url.QueryEscape(query)) + } + + var items []GetTemplatesResponse + httpReq, err := retryablehttp.NewRequest(http.MethodGet, builder.String(), nil) + if err != nil { + return items, errors.Wrap(err, "could not make request") + } + + resp, err := c.sendRequest(httpReq) + if err != nil { + return nil, errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + + if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil { + return items, errors.Wrap(err, "could not decode results") + } + return items, nil +} + +func (c *Client) RemoveDatasource(datasource int64, name string) error { + var builder strings.Builder + _, _ = builder.WriteString(c.baseURL) + _, _ = builder.WriteString("/datasources") + + if name != "" { + _, _ = builder.WriteString("?name=") + _, _ = builder.WriteString(name) + } else if datasource != 0 { + _, _ = builder.WriteString("?id=") + _, _ = builder.WriteString(strconv.FormatInt(datasource, 10)) + } + + httpReq, err := retryablehttp.NewRequest(http.MethodDelete, builder.String(), nil) + if err != nil { + return errors.Wrap(err, "could not make request") + } + + resp, err := c.sendRequest(httpReq) + if err != nil { + return errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + return nil +} + +func (c *Client) AddTemplate(name, contents string) (string, error) { + file, err := os.Open(contents) + if err != nil { + return "", errors.Wrap(err, "could not open contents") + } + defer file.Close() + + var buf bytes.Buffer + writer := multipart.NewWriter(&buf) + _ = writer.WriteField("name", name) + fileWriter, _ := writer.CreateFormFile("file", filepath.Base(contents)) + _, _ = io.Copy(fileWriter, file) + _ = writer.Close() + + httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/templates", c.baseURL), &buf) + if err != nil { + return "", errors.Wrap(err, "could not make request") + } + httpReq.Header.Set("Content-Type", writer.FormDataContentType()) + + resp, err := c.sendRequest(httpReq) + if err != nil { + return "", errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + + var item AddItemResponse + if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil { + return "", errors.Wrap(err, "could not decode results") + } + return item.Ok, nil +} + +func (c *Client) AddTarget(name, contents string) (string, error) { + file, err := os.Open(contents) + if err != nil { + return "", errors.Wrap(err, "could not open contents") + } + defer file.Close() + + var buf bytes.Buffer + writer := multipart.NewWriter(&buf) + _ = writer.WriteField("name", name) + fileWriter, _ := writer.CreateFormFile("file", filepath.Base(contents)) + _, _ = io.Copy(fileWriter, file) + _ = writer.Close() + + httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/targets", c.baseURL), &buf) + if err != nil { + return "", errors.Wrap(err, "could not make request") + } + httpReq.Header.Set("Content-Type", writer.FormDataContentType()) + + resp, err := c.sendRequest(httpReq) + if err != nil { + return "", errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + + var item AddItemResponse + if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil { + return "", errors.Wrap(err, "could not decode results") + } + return item.Ok, nil +} + +func (c *Client) RemoveTemplate(ID int64) error { + httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/templates/%d", c.baseURL, ID), nil) + if err != nil { + return errors.Wrap(err, "could not make request") + } + + resp, err := c.sendRequest(httpReq) + if err != nil { + return errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + return nil +} + +func (c *Client) RemoveTarget(ID int64) error { + httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/targets/%d", c.baseURL, ID), nil) + if err != nil { + return errors.Wrap(err, "could not make request") + } + + resp, err := c.sendRequest(httpReq) + if err != nil { + return errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + return nil +} + +func (c *Client) GetTarget(ID int64) (io.ReadCloser, error) { + httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/targets/%d", c.baseURL, ID), nil) + if err != nil { + return nil, errors.Wrap(err, "could not make request") + } + + resp, err := c.sendRequest(httpReq) + if err != nil { + return nil, errors.Wrap(err, "could not do request") + } + return resp.Body, nil +} + +func (c *Client) GetTemplate(ID int64) (io.ReadCloser, error) { + httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/templates/%d", c.baseURL, ID), nil) + if err != nil { + return nil, errors.Wrap(err, "could not make request") + } + + resp, err := c.sendRequest(httpReq) + if err != nil { + return nil, errors.Wrap(err, "could not do request") + } + return resp.Body, nil +} + +func (c *Client) ExistsTarget(id int64) (ExistsInputResponse, error) { + var item ExistsInputResponse + httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/targets/%d/exists", c.baseURL, id), nil) + if err != nil { + return item, errors.Wrap(err, "could not make request") + } + + resp, err := c.sendRequest(httpReq) + if err != nil { + return item, errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + + if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil { + return item, errors.Wrap(err, "could not decode results") + } + return item, nil +} + +func (c *Client) ExistsTemplate(id int64) (ExistsInputResponse, error) { + var item ExistsInputResponse + httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/templates/%d/exists", c.baseURL, id), nil) + if err != nil { + return item, errors.Wrap(err, "could not make request") + } + + resp, err := c.sendRequest(httpReq) + if err != nil { + return item, errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + + if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil { + return item, errors.Wrap(err, "could not decode results") + } + return item, nil +} + +const apiKeyParameter = "X-API-Key" + +type errorResponse struct { + Message string `json:"message"` +} + +func (c *Client) sendRequest(req *retryablehttp.Request) (*http.Response, error) { + req.Header.Set(apiKeyParameter, c.apiKey) + + resp, err := c.httpclient.Do(req) + if err != nil { + return nil, errors.Wrap(err, "could not do request") + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest { + data, _ := io.ReadAll(resp.Body) + resp.Body.Close() + var errRes errorResponse + if err = json.NewDecoder(bytes.NewReader(data)).Decode(&errRes); err == nil { + return nil, errors.New(errRes.Message) + } + return nil, fmt.Errorf("unknown error, status code: %d=%s", resp.StatusCode, string(data)) + } + return resp, nil +} diff --git a/v2/internal/runner/nucleicloud/types.go b/v2/internal/runner/nucleicloud/types.go index 889b003d..a4a292c2 100644 --- a/v2/internal/runner/nucleicloud/types.go +++ b/v2/internal/runner/nucleicloud/types.go @@ -2,6 +2,9 @@ package nucleicloud import ( "time" + + "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) // AddScanRequest is a nuclei scan input item. @@ -13,7 +16,32 @@ type AddScanRequest struct { // PrivateTemplates is a map of template-name->contents that // are private to the user executing the scan. (TODO: TBD) PrivateTemplates map[string]string `json:"private_templates,omitempty"` - IsTemporary bool `json:"is_temporary"` + // CloudTargets is a list of cloud targets for the scan + CloudTargets []string `json:"cloud_targets,omitempty"` + // CloudTemplates is a list of cloud templates for the scan + CloudTemplates []string `json:"cloud_templates,omitempty"` + // Filtering contains optional filtering options for scan additions + Filtering *AddScanRequestConfiguration `json:"filtering"` + + IsTemporary bool `json:"is_temporary"` +} + +// AddScanRequestConfiguration contains filtering options for scan addition +type AddScanRequestConfiguration struct { + Authors []string `json:"author,omitempty"` + Tags []string `json:"tags,omitempty"` + ExcludeTags []string `json:"exclude-tags,omitempty"` + IncludeTags []string `json:"include-tags,omitempty"` + IncludeIds []string `json:"include-ids,omitempty"` + ExcludeIds []string `json:"exclude-ids,omitempty"` + IncludeTemplates []string `json:"include-templates,omitempty"` + ExcludedTemplates []string `json:"exclude-templates,omitempty"` + ExcludeMatchers []string `json:"exclude-matchers,omitempty"` + Severities severity.Severities `json:"severities,omitempty"` + ExcludeSeverities severity.Severities `json:"exclude-severities,omitempty"` + Protocols types.ProtocolTypes `json:"protocols,omitempty"` + ExcludeProtocols types.ProtocolTypes `json:"exclude-protocols,omitempty"` + IncludeConditions []string `json:"include-conditions,omitempty"` } type GetResultsResponse struct { @@ -22,7 +50,7 @@ type GetResultsResponse struct { } type GetScanRequest struct { - Id string `json:"id"` + Id int64 `json:"id"` Total int32 `json:"total"` Current int32 `json:"current"` Finished bool `json:"finished"` @@ -33,6 +61,13 @@ type GetScanRequest struct { Matches int64 `json:"matches"` } +// AddDataSourceResponse is a add data source response item. +type AddDataSourceResponse struct { + ID int64 `json:"id"` + Hash string `json:"hash"` + Secret string `json:"secret,omitempty"` +} + type GetResultsResponseItem struct { ID int64 `json:"id"` Raw string `json:"raw"` @@ -41,3 +76,79 @@ type GetResultsResponseItem struct { type DeleteScanResults struct { OK bool `json:"ok"` } + +// StatusDataSourceRequest is a add data source request item. +type StatusDataSourceRequest struct { + Repo string `json:"repo"` + Token string `json:"token"` +} + +// StatusDataSourceResponse is a add data source response item. +type StatusDataSourceResponse struct { + ID int64 `json:"id"` +} + +// AddDataSourceRequest is a add data source request item. +type AddDataSourceRequest struct { + Type string `json:"type"` + Repo string `json:"repo"` + Token string `json:"token"` + Sync bool `json:"sync"` +} + +// ExistsDataSourceItemRequest is a request to identify whether a data +// source item exists. +type ExistsDataSourceItemRequest struct { + Type string `json:"type"` + Contents string `json:"contents"` +} + +// GetDataSourceResponse is response for a get data source request +type GetDataSourceResponse struct { + ID int64 `json:"id"` + Hash string `json:"hash"` + Type string `json:"type"` + Path string `json:"path"` + Repo string `json:"repo"` + Updatedat time.Time `json:"updated_at"` +} + +// GetTargetResponse is the response for a get target request +type GetTargetResponse struct { + ID int64 `json:"id"` + DataSource int64 `json:"data_source"` + Name string `json:"name"` + Reference string `json:"reference"` + Count int64 `json:"count"` + Hash string `json:"hash"` + Type string `json:"type"` +} + +// GetTemplatesResponse is the response for a get templates request +type GetTemplatesResponse struct { + ID int64 `json:"id,omitempty"` + DataSource int64 `json:"data_source,omitempty"` + Name string `json:"name,omitempty"` + Reference string `json:"reference,omitempty"` + Hash string `json:"hash,omitempty"` + Type string `json:"type,omitempty"` +} + +// AddItemResponse is the response to add item request +type AddItemResponse struct { + Ok string `json:"ok"` +} + +type ListScanOutput struct { + Timestamp string `json:"timestamp"` + ScanID int64 `json:"scan_id"` + ScanTime string `json:"scan_time"` + ScanResult int `json:"scan_result"` + ScanStatus string `json:"scan_status"` + Target int `json:"target"` + Template int `json:"template"` +} + +type ExistsInputResponse struct { + Reference string `json:"reference"` +} diff --git a/v2/internal/runner/nucleicloud/utils.go b/v2/internal/runner/nucleicloud/utils.go index 611e4afa..77a87573 100644 --- a/v2/internal/runner/nucleicloud/utils.go +++ b/v2/internal/runner/nucleicloud/utils.go @@ -5,10 +5,13 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" ) +const DDMMYYYYhhmmss = "2006-01-02 15:04:05" + // ReadCatalogChecksum reads catalog checksum from nuclei-templates repository func ReadCatalogChecksum() map[string]string { config, _ := config.ReadConfiguration() @@ -38,3 +41,29 @@ func ReadCatalogChecksum() map[string]string { } return checksums } + +func PrepareScanListOutput(v GetScanRequest) ListScanOutput { + output := ListScanOutput{} + loc, _ := time.LoadLocation("Local") + status := "finished" + + t := v.FinishedAt + duration := t.Sub(v.CreatedAt) + + if !v.Finished { + status = "running" + t = time.Now().UTC() + duration = t.Sub(v.CreatedAt).Round(60 * time.Second) + } + + val := v.CreatedAt.In(loc).Format(DDMMYYYYhhmmss) + + output.Timestamp = val + output.ScanID = v.Id + output.ScanTime = duration.String() + output.ScanResult = int(v.Matches) + output.ScanStatus = status + output.Target = int(v.Targets) + output.Template = int(v.Templates) + return output +} diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 88595f06..8cace994 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -160,16 +160,7 @@ func validateOptions(options *types.Options) error { } // Verify aws secrets are passed if s3 template bucket passed if options.AwsBucketName != "" && options.UpdateTemplates { - var missing []string - if options.AwsAccessKey == "" { - missing = append(missing, "AWS_ACCESS_KEY") - } - if options.AwsSecretKey == "" { - missing = append(missing, "AWS_SECRET_KEY") - } - if options.AwsRegion == "" { - missing = append(missing, "AWS_REGION") - } + missing := validateMissingS3Options(options) if missing != nil { return fmt.Errorf("aws s3 bucket details are missing. Please provide %s", strings.Join(missing, ",")) } @@ -195,9 +186,63 @@ func validateOptions(options *types.Options) error { return errors.New("ipv4 and/or ipv6 must be selected") } + // Validate cloud option + if err := validateCloudOptions(options); err != nil { + return err + } return nil } +func validateCloudOptions(options *types.Options) error { + if options.HasCloudOptions() && !options.Cloud { + return errors.New("cloud flags cannot be used without cloud option") + } + if options.Cloud { + if options.CloudAPIKey == "" { + return errors.New("missing NUCLEI_CLOUD_API env variable") + } + var missing []string + switch options.AddDatasource { + case "s3": + missing = validateMissingS3Options(options) + case "github": + missing = validateMissingGithubOptions(options) + } + if len(missing) > 0 { + return fmt.Errorf("missing %v env variables", strings.Join(missing, ", ")) + } + } + return nil +} + +func validateMissingS3Options(options *types.Options) []string { + var missing []string + if options.AwsBucketName == "" { + missing = append(missing, "AWS_TEMPLATE_BUCKET") + } + if options.AwsAccessKey == "" { + missing = append(missing, "AWS_ACCESS_KEY") + } + if options.AwsSecretKey == "" { + missing = append(missing, "AWS_SECRET_KEY") + } + if options.AwsRegion == "" { + missing = append(missing, "AWS_REGION") + } + return missing +} + +func validateMissingGithubOptions(options *types.Options) []string { + var missing []string + if options.GithubToken == "" { + missing = append(missing, "GITHUB_TOKEN") + } + if len(options.GithubTemplateRepo) == 0 { + missing = append(missing, "GITHUB_TEMPLATE_REPO") + } + return missing +} + // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output @@ -275,6 +320,14 @@ func validateCertificatePaths(certificatePaths []string) { // Read the input from env and set options func readEnvInputVars(options *types.Options) { + if strings.EqualFold(os.Getenv("NUCLEI_CLOUD"), "true") { + options.Cloud = true + } + if options.CloudURL = os.Getenv("NUCLEI_CLOUD_SERVER"); options.CloudURL == "" { + options.CloudURL = "https://cloud-dev.nuclei.sh" + } + options.CloudAPIKey = os.Getenv("NUCLEI_CLOUD_API") + options.GithubToken = os.Getenv("GITHUB_TOKEN") repolist := os.Getenv("GITHUB_TEMPLATE_REPO") if repolist != "" { diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 0d67e367..3f140883 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -11,6 +11,7 @@ import ( _ "net/http/pprof" "os" "path/filepath" + "strconv" "strings" "time" @@ -77,6 +78,7 @@ type Runner struct { pprofServer *http.Server customTemplates []customtemplates.Provider cloudClient *nucleicloud.Client + cloudTargets []string } const pprofServerAddress = "127.0.0.1:8086" @@ -174,12 +176,29 @@ func New(options *types.Options) (*Runner, error) { }() } - if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates { + if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && (options.UpdateTemplates && !options.Cloud) { os.Exit(0) } // Initialize the input source - hmapInput, err := hybrid.New(options) + hmapInput, err := hybrid.New(&hybrid.Options{ + Options: options, + NotFoundCallback: func(target string) bool { + parsed, parseErr := strconv.ParseInt(target, 10, 64) + if parseErr != nil { + if err := runner.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Contents: target, Type: "targets"}); err == nil { + runner.cloudTargets = append(runner.cloudTargets, target) + return true + } + return false + } + if exists, err := runner.cloudClient.ExistsTarget(parsed); err == nil { + runner.cloudTargets = append(runner.cloudTargets, exists.Reference) + return true + } + return false + }, + }) if err != nil { return nil, errors.Wrap(err, "could not create input provider") } @@ -401,6 +420,26 @@ func (r *Runner) RunEnumeration() error { if err != nil { return errors.Wrap(err, "could not load templates from config") } + + var cloudTemplates []string + if r.options.Cloud { + // hook template loading + store.NotFoundCallback = func(template string) bool { + parsed, parseErr := strconv.ParseInt(template, 10, 64) + if parseErr != nil { + if err := r.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Type: "templates", Contents: template}); err == nil { + cloudTemplates = append(cloudTemplates, template) + return true + } + return false + } + if exists, err := r.cloudClient.ExistsTemplate(parsed); err == nil { + cloudTemplates = append(cloudTemplates, exists.Reference) + return true + } + return false + } + } if r.options.Validate { if err := store.ValidateTemplates(); err != nil { return err @@ -443,14 +482,36 @@ func (r *Runner) RunEnumeration() error { var results *atomic.Bool if r.options.Cloud { if r.options.ScanList { - err = r.getScanList() + err = r.getScanList(r.options.OutputLimit) } else if r.options.DeleteScan != "" { err = r.deleteScan(r.options.DeleteScan) } else if r.options.ScanOutput != "" { - err = r.getResults(r.options.ScanOutput) + err = r.getResults(r.options.ScanOutput, r.options.OutputLimit) + } else if r.options.ListDatasources { + err = r.listDatasources() + } else if r.options.ListTargets { + err = r.listTargets() + } else if r.options.ListTemplates { + err = r.listTemplates() + } else if r.options.AddDatasource != "" { + err = r.addCloudDataSource(r.options.AddDatasource) + } else if r.options.RemoveDatasource != "" { + err = r.removeDatasource(r.options.RemoveDatasource) + } else if r.options.AddTarget != "" { + err = r.addTarget(r.options.AddTarget) + } else if r.options.AddTemplate != "" { + err = r.addTemplate(r.options.AddTemplate) + } else if r.options.GetTarget != "" { + err = r.getTarget(r.options.GetTarget) + } else if r.options.GetTemplate != "" { + err = r.getTemplate(r.options.GetTemplate) + } else if r.options.RemoveTarget != "" { + err = r.removeTarget(r.options.RemoveTarget) + } else if r.options.RemoveTemplate != "" { + err = r.removeTemplate(r.options.RemoveTemplate) } else { gologger.Info().Msgf("Running scan on cloud with URL %s", r.options.CloudURL) - results, err = r.runCloudEnumeration(store, r.options.NoStore) + results, err = r.runCloudEnumeration(store, cloudTemplates, r.cloudTargets, r.options.NoStore, r.options.OutputLimit) enumeration = true } } else { @@ -615,8 +676,9 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { if len(store.Workflows()) > 0 { gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows())) } - - gologger.Info().Msgf("Targets loaded for scan: %d", r.hmapInputProvider.Count()) + if r.hmapInputProvider.Count() > 0 { + gologger.Info().Msgf("Targets loaded for scan: %d", r.hmapInputProvider.Count()) + } } func (r *Runner) readNewTemplatesWithVersionFile(version string) ([]string, error) { diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 5a7f8228..4942753e 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -86,7 +86,7 @@ func (r *Runner) updateTemplates() error { // TODO this method does more than ju } // if disable update check flag is passed and no update template flag is passed - if r.options.NoUpdateTemplates && !r.options.UpdateTemplates { + if (r.options.NoUpdateTemplates && !r.options.UpdateTemplates) || r.options.Cloud { return nil } diff --git a/v2/pkg/catalog/catalogue.go b/v2/pkg/catalog/catalogue.go index 3d20752b..7fd41c26 100644 --- a/v2/pkg/catalog/catalogue.go +++ b/v2/pkg/catalog/catalogue.go @@ -12,7 +12,7 @@ type Catalog interface { // or folders provided as in. GetTemplatePath(target string) ([]string, error) // GetTemplatesPath returns a list of absolute paths for the provided template list. - GetTemplatesPath(definitions []string) []string + GetTemplatesPath(definitions []string) ([]string, map[string]error) // ResolvePath resolves the path to an absolute one in various ways. // // It checks if the filename is an absolute path, looks in the current directory diff --git a/v2/pkg/catalog/disk/find.go b/v2/pkg/catalog/disk/find.go index c1f9727b..9c1dff46 100644 --- a/v2/pkg/catalog/disk/find.go +++ b/v2/pkg/catalog/disk/find.go @@ -7,14 +7,14 @@ import ( "strings" "github.com/pkg/errors" - "github.com/projectdiscovery/gologger" ) // GetTemplatesPath returns a list of absolute paths for the provided template list. -func (c *DiskCatalog) GetTemplatesPath(definitions []string) []string { +func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[string]error) { // keeps track of processed dirs and files processed := make(map[string]bool) allTemplates := []string{} + erred := make(map[string]error) for _, t := range definitions { if strings.HasPrefix(t, "http") && (strings.HasSuffix(t, ".yaml") || strings.HasSuffix(t, ".yml")) { @@ -25,7 +25,7 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) []string { } else { paths, err := c.GetTemplatePath(t) if err != nil { - gologger.Error().Msgf("Could not find template '%s': %s\n", t, err) + erred[t] = err } for _, path := range paths { if _, ok := processed[path]; !ok { @@ -35,7 +35,7 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) []string { } } } - return allTemplates + return allTemplates, erred } // GetTemplatePath parses the specified input template path and returns a compiled diff --git a/v2/pkg/catalog/disk/path.go b/v2/pkg/catalog/disk/path.go index f1bc3ba9..9331ac5a 100644 --- a/v2/pkg/catalog/disk/path.go +++ b/v2/pkg/catalog/disk/path.go @@ -6,8 +6,6 @@ import ( "path/filepath" "github.com/pkg/errors" - - folderutil "github.com/projectdiscovery/utils/folder" ) // ResolvePath resolves the path to an absolute one in various ways. @@ -49,20 +47,8 @@ var errNoValidCombination = errors.New("no valid combination found") // tryResolve attempts to load locate the target by iterating across all the folders tree func (c *DiskCatalog) tryResolve(fullPath string) (string, error) { - dir, filename := filepath.Split(fullPath) - pathInfo, err := folderutil.NewPathInfo(dir) - if err != nil { - return "", err + if _, err := os.Stat(fullPath); !os.IsNotExist(err) { + return fullPath, nil } - pathInfoItems, err := pathInfo.MeshWith(filename) - if err != nil { - return "", err - } - for _, pathInfoItem := range pathInfoItems { - if _, err := os.Stat(pathInfoItem); !os.IsNotExist(err) { - return pathInfoItem, nil - } - } - return "", errNoValidCombination } diff --git a/v2/pkg/catalog/loader/filter/path_filter.go b/v2/pkg/catalog/loader/filter/path_filter.go index d9b7ba01..00f5b3b2 100644 --- a/v2/pkg/catalog/loader/filter/path_filter.go +++ b/v2/pkg/catalog/loader/filter/path_filter.go @@ -18,12 +18,13 @@ type PathFilterConfig struct { // NewPathFilter creates a new path filter from provided config func NewPathFilter(config *PathFilterConfig, catalogClient catalog.Catalog) *PathFilter { + paths, _ := catalogClient.GetTemplatesPath(config.ExcludedTemplates) filter := &PathFilter{ - excludedTemplates: catalogClient.GetTemplatesPath(config.ExcludedTemplates), + excludedTemplates: paths, alwaysIncludedTemplatesMap: make(map[string]struct{}), } - alwaysIncludeTemplates := catalogClient.GetTemplatesPath(config.IncludedTemplates) + alwaysIncludeTemplates, _ := catalogClient.GetTemplatesPath(config.IncludedTemplates) for _, tpl := range alwaysIncludeTemplates { filter.alwaysIncludedTemplatesMap[tpl] = struct{}{} } diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 26b7a864..81f7c6bc 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -58,6 +58,10 @@ type Store struct { workflows []*templates.Template preprocessor templates.Preprocessor + + // NotFoundCallback is called for each not found template + // This overrides error handling for not found templatesss + NotFoundCallback func(template string) bool } // NewConfig returns a new loader config @@ -174,8 +178,10 @@ func init() { // ValidateTemplates takes a list of templates and validates them // erroring out on discovering any faulty templates. func (store *Store) ValidateTemplates() error { - templatePaths := store.config.Catalog.GetTemplatesPath(store.finalTemplates) - workflowPaths := store.config.Catalog.GetTemplatesPath(store.finalWorkflows) + templatePaths, errs := store.config.Catalog.GetTemplatesPath(store.finalTemplates) + store.logErroredTemplates(errs) + workflowPaths, errs := store.config.Catalog.GetTemplatesPath(store.finalWorkflows) + store.logErroredTemplates(errs) filteredTemplatePaths := store.pathFilter.Match(templatePaths) filteredWorkflowPaths := store.pathFilter.Match(workflowPaths) @@ -263,7 +269,8 @@ func isParsingError(message string, template string, err error) bool { // LoadTemplates takes a list of templates and returns paths for them func (store *Store) LoadTemplates(templatesList []string) []*templates.Template { - includedTemplates := store.config.Catalog.GetTemplatesPath(templatesList) + includedTemplates, errs := store.config.Catalog.GetTemplatesPath(templatesList) + store.logErroredTemplates(errs) templatePathMap := store.pathFilter.Match(includedTemplates) loadedTemplates := make([]*templates.Template, 0, len(templatePathMap)) @@ -295,7 +302,8 @@ func (store *Store) LoadTemplates(templatesList []string) []*templates.Template // LoadWorkflows takes a list of workflows and returns paths for them func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template { - includedWorkflows := store.config.Catalog.GetTemplatesPath(workflowsList) + includedWorkflows, errs := store.config.Catalog.GetTemplatesPath(workflowsList) + store.logErroredTemplates(errs) workflowPathMap := store.pathFilter.Match(includedWorkflows) loadedWorkflows := make([]*templates.Template, 0, len(workflowPathMap)) @@ -319,7 +327,8 @@ func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template // LoadTemplatesWithTags takes a list of templates and extra tags // returning templates that match. func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templates.Template { - includedTemplates := store.config.Catalog.GetTemplatesPath(templatesList) + includedTemplates, errs := store.config.Catalog.GetTemplatesPath(templatesList) + store.logErroredTemplates(errs) templatePathMap := store.pathFilter.Match(includedTemplates) loadedTemplates := make([]*templates.Template, 0, len(templatePathMap)) @@ -382,3 +391,11 @@ func workflowContainsProtocol(workflow []*workflows.WorkflowTemplate) bool { } return false } + +func (s *Store) logErroredTemplates(erred map[string]error) { + for template, err := range erred { + if s.NotFoundCallback == nil || !s.NotFoundCallback(template) { + gologger.Error().Msgf("Could not find template '%s': %s", template, err) + } + } +} diff --git a/v2/pkg/core/inputs/hybrid/hmap.go b/v2/pkg/core/inputs/hybrid/hmap.go index e6061b6c..3937e283 100644 --- a/v2/pkg/core/inputs/hybrid/hmap.go +++ b/v2/pkg/core/inputs/hybrid/hmap.go @@ -41,9 +41,20 @@ type Input struct { sync.Once } +// Options is a wrapper around types.Options structure +type Options struct { + // Options contains options for hmap provider + Options *types.Options + // NotFoundCallback is called for each not found target + // This overrides error handling for not found target + NotFoundCallback func(template string) bool +} + // New creates a new hmap backed nuclei Input Provider // and initializes it based on the passed options Model. -func New(options *types.Options) (*Input, error) { +func New(opts *Options) (*Input, error) { + options := opts.Options + hm, err := hybrid.New(hybrid.DefaultDiskOptions) if err != nil { return nil, errors.Wrap(err, "could not create temporary input file") @@ -71,7 +82,7 @@ func New(options *types.Options) (*Input, error) { } input.hostMapStream = fkv } - if initErr := input.initializeInputSources(options); initErr != nil { + if initErr := input.initializeInputSources(opts); initErr != nil { return nil, initErr } if input.dupeCount > 0 { @@ -89,7 +100,9 @@ func (i *Input) Close() { } // initializeInputSources initializes the input sources for hmap input -func (i *Input) initializeInputSources(options *types.Options) error { +func (i *Input) initializeInputSources(opts *Options) error { + options := opts.Options + // Handle targets flags for _, target := range options.Targets { switch { @@ -111,11 +124,15 @@ func (i *Input) initializeInputSources(options *types.Options) error { if options.TargetsFilePath != "" { input, inputErr := os.Open(options.TargetsFilePath) if inputErr != nil { - return errors.Wrap(inputErr, "could not open targets file") + // Handle cloud based input here. + if opts.NotFoundCallback == nil || !opts.NotFoundCallback(options.TargetsFilePath) { + return errors.Wrap(inputErr, "could not open targets file") + } + } + if input != nil { + i.scanInputFromReader(input) + input.Close() } - defer input.Close() - - i.scanInputFromReader(input) } if options.Uncover && options.UncoverQuery != nil { gologger.Info().Msgf("Running uncover query against: %s", strings.Join(options.UncoverEngine, ",")) diff --git a/v2/pkg/external/customtemplates/github.go b/v2/pkg/external/customtemplates/github.go index 326ab1c8..550d5fe0 100644 --- a/v2/pkg/external/customtemplates/github.go +++ b/v2/pkg/external/customtemplates/github.go @@ -30,7 +30,7 @@ func (customTemplate *customTemplateGithubRepo) Download(location string, ctx co if !fileutil.FolderExists(clonePath) { err := customTemplate.cloneRepo(clonePath, customTemplate.githubToken) if err != nil { - gologger.Info().Msgf("%s", err) + gologger.Error().Msgf("%s", err) } else { gologger.Info().Msgf("Repo %s/%s cloned successfully at %s", customTemplate.owner, customTemplate.reponame, clonePath) } @@ -49,7 +49,7 @@ func (customTemplate *customTemplateGithubRepo) Update(location string, ctx cont } err := customTemplate.pullChanges(clonePath, customTemplate.githubToken) if err != nil { - gologger.Info().Msgf("%s", err) + gologger.Error().Msgf("%s", err) } else { gologger.Info().Msgf("Repo %s/%s successfully pulled the changes.\n", customTemplate.owner, customTemplate.reponame) } diff --git a/v2/pkg/external/customtemplates/templates_provider.go b/v2/pkg/external/customtemplates/templates_provider.go index de6a8e97..e4b5f0ed 100644 --- a/v2/pkg/external/customtemplates/templates_provider.go +++ b/v2/pkg/external/customtemplates/templates_provider.go @@ -21,18 +21,21 @@ type Provider interface { // parseCustomTemplates function reads the options.GithubTemplateRepo list, // Checks the given repos are valid or not and stores them into runner.CustomTemplates func ParseCustomTemplates(options *types.Options) []Provider { + if options.Cloud { + return nil + } var customTemplates []Provider gitHubClient := getGHClientIncognito() for _, repoName := range options.GithubTemplateRepo { owner, repo, err := getOwnerAndRepo(repoName) if err != nil { - gologger.Info().Msgf("%s", err) + gologger.Error().Msgf("%s", err) continue } githubRepo, err := getGithubRepo(gitHubClient, owner, repo, options.GithubToken) if err != nil { - gologger.Info().Msgf("%s", err) + gologger.Error().Msgf("%s", err) continue } customTemplateRepo := &customTemplateGithubRepo{ diff --git a/v2/pkg/model/types/severity/severities.go b/v2/pkg/model/types/severity/severities.go index f17811da..f45fba8c 100644 --- a/v2/pkg/model/types/severity/severities.go +++ b/v2/pkg/model/types/severity/severities.go @@ -68,6 +68,14 @@ func (severities Severities) String() string { return strings.Join(stringSeverities, ", ") } +func (severities Severities) MarshalJSON() ([]byte, error) { + var stringSeverities = make([]string, 0, len(severities)) + for _, severity := range severities { + stringSeverities = append(stringSeverities, severity.String()) + } + return json.Marshal(stringSeverities) +} + func setSeverity(severities *Severities, value string) error { computedSeverity, err := toSeverity(value) if err != nil { diff --git a/v2/pkg/model/types/severity/severity_test.go b/v2/pkg/model/types/severity/severity_test.go index 30ff4be9..30abb69c 100644 --- a/v2/pkg/model/types/severity/severity_test.go +++ b/v2/pkg/model/types/severity/severity_test.go @@ -6,6 +6,7 @@ import ( "gopkg.in/yaml.v2" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestYamlUnmarshal(t *testing.T) { @@ -65,3 +66,12 @@ func unmarshal(value string, unmarshaller func(data []byte, v interface{}) error func createYAML(value string) string { return "severity: " + value + "\n" } + +func TestMarshalJSON(t *testing.T) { + unmarshalled := Severities{Low, Medium} + data, err := unmarshalled.MarshalJSON() + if err != nil { + panic(err) + } + require.Equal(t, "[\"low\",\"medium\"]", string(data), "could not marshal json") +} diff --git a/v2/pkg/parsers/workflow_loader.go b/v2/pkg/parsers/workflow_loader.go index 6c1dab89..d450b34d 100644 --- a/v2/pkg/parsers/workflow_loader.go +++ b/v2/pkg/parsers/workflow_loader.go @@ -40,7 +40,10 @@ func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error) } func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string { - includedTemplates := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory}) + includedTemplates, errs := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory}) + for template, err := range errs { + gologger.Error().Msgf("Could not find template '%s': %s", template, err) + } templatePathMap := w.pathFilter.Match(includedTemplates) loadedTemplates := make([]string, 0, len(templatePathMap)) @@ -56,7 +59,10 @@ func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string } func (w *workflowLoader) GetTemplatePaths(templatesList []string, noValidate bool) []string { - includedTemplates := w.options.Catalog.GetTemplatesPath(templatesList) + includedTemplates, errs := w.options.Catalog.GetTemplatesPath(templatesList) + for template, err := range errs { + gologger.Error().Msgf("Could not find template '%s': %s", template, err) + } templatesPathMap := w.pathFilter.Match(includedTemplates) loadedTemplates := make([]string, 0, len(templatesPathMap)) diff --git a/v2/pkg/templates/types/types.go b/v2/pkg/templates/types/types.go index 85fc3bb9..f7ae8cd3 100644 --- a/v2/pkg/templates/types/types.go +++ b/v2/pkg/templates/types/types.go @@ -154,6 +154,14 @@ func (protocolTypes *ProtocolTypes) UnmarshalYAML(unmarshal func(interface{}) er return nil } +func (protocolTypes ProtocolTypes) MarshalJSON() ([]byte, error) { + var stringProtocols = make([]string, 0, len(protocolTypes)) + for _, protocol := range protocolTypes { + stringProtocols = append(stringProtocols, protocol.String()) + } + return json.Marshal(stringProtocols) +} + func (protocolTypes ProtocolTypes) String() string { var stringTypes []string for _, t := range protocolTypes { diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 9d5e9ed0..6fc743ed 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -97,10 +97,34 @@ type Options struct { CloudAPIKey string // Scanlist feature to get all the scan ids for a user ScanList bool + // ListDatasources enables listing of datasources for user + ListDatasources bool + // ListTargets enables listing of targets for user + ListTargets bool + // ListTemplates enables listing of templates for user + ListTemplates bool + // Limit the number of items at a time + OutputLimit int // Nostore NoStore bool // Delete scan DeleteScan string + // AddDatasource adds a datasource to cloud storage + AddDatasource string + // RemoveDatasource deletes a datasource from cloud storage + RemoveDatasource string + // AddTemplate adds a list of templates to custom datasource + AddTemplate string + // AddTarget adds a list of targets to custom datasource + AddTarget string + // GetTemplate gets a template by id + GetTemplate string + // GetTarget gets a target by id + GetTarget string + // RemoveTemplate removes a list of templates + RemoveTemplate string + // RemoveTarget removes a list of targets + RemoveTarget string // Get issues for a scan ScanOutput string // ResolversFile is a file containing resolvers for nuclei. @@ -157,6 +181,8 @@ type Options struct { Headless bool // ShowBrowser specifies whether the show the browser in headless mode ShowBrowser bool + // NoTables disables pretty printing of cloud results in tables + NoTables bool // DisableClustering disables clustering of templates DisableClustering bool // UseInstalledChrome skips chrome install and use local instance @@ -352,3 +378,8 @@ func DefaultOptions() *Options { MaxHostError: 30, } } + +// HasCloudOptions returns true if cloud options have been specified +func (options *Options) HasCloudOptions() bool { + return options.ScanList || options.DeleteScan != "" || options.ScanOutput != "" || options.ListDatasources || options.ListTargets || options.ListTemplates || options.RemoveDatasource != "" || options.AddTarget != "" || options.AddTemplate != "" || options.RemoveTarget != "" || options.RemoveTemplate != "" || options.GetTarget != "" || options.GetTemplate != "" +}