diff --git a/cmd/integration-test/integration-test.go b/cmd/integration-test/integration-test.go index 697269b8..672224e1 100644 --- a/cmd/integration-test/integration-test.go +++ b/cmd/integration-test/integration-test.go @@ -174,6 +174,7 @@ func execute(testCase testutils.TestCase, templatePath string) (string, error) { } func expectResultsCount(results []string, expectedNumbers ...int) error { + results = filterHeadlessLogs(results) match := sliceutil.Contains(expectedNumbers, len(results)) if !match { return fmt.Errorf("incorrect number of results: %d (actual) vs %v (expected) \nResults:\n\t%s\n", len(results), expectedNumbers, strings.Join(results, "\n\t")) // nolint:all @@ -186,3 +187,17 @@ func normalizeSplit(str string) []string { return r == ',' }) } + +// if chromium is not installed go-rod installs it in .cache directory +// this function filters out the logs from download and installation +func filterHeadlessLogs(results []string) []string { + // [launcher.Browser] 2021/09/23 15:24:05 [launcher] [info] Starting browser + filtered := []string{} + for _, result := range results { + if strings.Contains(result, "[launcher.Browser]") { + continue + } + filtered = append(filtered, result) + } + return filtered +} diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 3f11416e..ed45947b 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -164,6 +164,7 @@ func readConfig() *goflags.FlagSet { // when true updates nuclei binary to latest version var updateNucleiBinary bool + var pdcpauth bool flagSet := goflags.NewFlagSet() flagSet.CaseSensitive = true @@ -365,27 +366,8 @@ on extensive configurability, massive extensibility and ease of use.`) ) flagSet.CreateGroup("cloud", "Cloud", - flagSet.BoolVar(&options.Cloud, "cloud", false, "run scan on nuclei cloud"), - flagSet.StringVarP(&options.AddDatasource, "add-datasource", "ads", "", "add specified data source (s3,github)"), - flagSet.StringVarP(&options.AddTarget, "add-target", "atr", "", "add target(s) to cloud"), - flagSet.StringVarP(&options.AddTemplate, "add-template", "atm", "", "add template(s) to cloud"), - flagSet.BoolVarP(&options.ScanList, "list-scan", "lsn", false, "list previous cloud scans"), - flagSet.StringVarP(&options.ScanOutput, "list-output", "lso", "", "list scan output by scan id"), - flagSet.BoolVarP(&options.ListTargets, "list-target", "ltr", false, "list cloud target by id"), - flagSet.BoolVarP(&options.ListTemplates, "list-template", "ltm", false, "list cloud template by id"), - flagSet.BoolVarP(&options.ListDatasources, "list-datasource", "lds", false, "list cloud datasource by id"), - flagSet.BoolVarP(&options.ListReportingSources, "list-reportsource", "lrs", false, "list reporting sources"), - flagSet.StringVarP(&options.DeleteScan, "delete-scan", "dsn", "", "delete cloud scan by id"), - flagSet.StringVarP(&options.RemoveTarget, "delete-target", "dtr", "", "delete target(s) from cloud"), - flagSet.StringVarP(&options.RemoveTemplate, "delete-template", "dtm", "", "delete template(s) from cloud"), - flagSet.StringVarP(&options.RemoveDatasource, "delete-datasource", "dds", "", "delete specified data source"), - flagSet.StringVarP(&options.DisableReportingSource, "disable-reportsource", "drs", "", "disable specified reporting source"), - flagSet.StringVarP(&options.EnableReportingSource, "enable-reportsource", "ers", "", "enable specified reporting source"), - flagSet.StringVarP(&options.GetTarget, "get-target", "gtr", "", "get target content by id"), - flagSet.StringVarP(&options.GetTemplate, "get-template", "gtm", "", "get template content by id"), - flagSet.BoolVarP(&options.NoStore, "no-store", "nos", false, "disable scan/output storage on cloud"), - flagSet.BoolVar(&options.NoTables, "no-tables", false, "do not display pretty-printed tables"), - flagSet.IntVar(&options.OutputLimit, "limit", 100, "limit the number of output to display"), + flagSet.BoolVar(&pdcpauth, "auth", false, "configure projectdiscovery cloud (pdcp) api key"), + flagSet.BoolVarP(&options.DisableCloudUpload, "disable-cloud-upload", "dcu", false, "disable uploading scan results to pdcp"), ) flagSet.SetCustomHelpText(`EXAMPLES: @@ -414,6 +396,10 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started goflags.DisableAutoConfigMigration = true _ = flagSet.Parse() + if pdcpauth { + runner.AuthWithPDCP() + } + gologger.DefaultLogger.SetTimestamp(options.Timestamp, levels.LevelDebug) if options.VerboseVerbose { diff --git a/go.mod b/go.mod index 1b471394..be781f2e 100644 --- a/go.mod +++ b/go.mod @@ -71,7 +71,6 @@ require ( github.com/go-sql-driver/mysql v1.6.0 github.com/h2non/filetype v1.1.3 github.com/hirochachacha/go-smb2 v1.1.0 - github.com/klauspost/compress v1.16.7 github.com/labstack/echo/v4 v4.10.2 github.com/lib/pq v1.10.1 github.com/mholt/archiver v3.1.1+incompatible @@ -163,6 +162,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kataras/jwt v0.1.10 // indirect + github.com/klauspost/compress v1.16.7 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mackerelio/go-osstat v0.2.4 // indirect diff --git a/internal/pdcp/auth.go b/internal/pdcp/auth.go new file mode 100644 index 00000000..c8776090 --- /dev/null +++ b/internal/pdcp/auth.go @@ -0,0 +1,76 @@ +// pdcp contains projectdiscovery cloud platform related features +// like result upload , dashboard etc. +package pdcp + +import ( + "fmt" + "os" + "strings" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" + "github.com/projectdiscovery/utils/env" + "golang.org/x/term" +) + +var ( + DashBoardURL = "https://cloud.projectdiscovery.io" + DefaultApiServer = "https://api.projectdiscovery.io" +) + +// CheckNValidateCredentials checks if credentials exist on filesystem +// if not waits for user to enter credentials and validates them +// and saves them to filesystem +// when validate is true any existing credentials are validated +// Note: this is meant to be used in cli only (interactive mode) +func CheckNValidateCredentials(toolName string) { + h := &PDCPCredHandler{} + creds, err := h.GetCreds() + if err == nil { + // validate by fetching user profile + gotCreds, err := h.ValidateAPIKey(creds.APIKey, creds.Server, config.BinaryName) + if err == nil { + gologger.Info().Msgf("You are logged in as (@%v)", gotCreds.Username) + os.Exit(0) + } + gologger.Error().Msgf("Invalid API key found in file, please recheck or recreate your API key and retry.") + } + if err != nil && err != ErrNoCreds { + // this is unexpected error log it + gologger.Error().Msgf("Could not read credentials from file: %s\n", err) + } + + // if we are here, we need to get credentials from user + gologger.Info().Msgf("Get your free api key by signing up at %v", DashBoardURL) + fmt.Printf("[*] Enter PDCP API Key (exit to abort): ") + bin, err := term.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + gologger.Fatal().Msgf("Could not read input from terminal: %s\n", err) + } + apiKey := string(bin) + if strings.EqualFold(apiKey, "exit") { + os.Exit(0) + } + fmt.Println() + // if env variable is set use that for validating api key + apiServer := env.GetEnvOrDefault(apiServerEnv, DefaultApiServer) + // validate by fetching user profile + validatedCreds, err := h.ValidateAPIKey(apiKey, apiServer, toolName) + if err == nil { + gologger.Info().Msgf("Successfully logged in as (@%v)", validatedCreds.Username) + if saveErr := h.SaveCreds(validatedCreds); saveErr != nil { + gologger.Warning().Msgf("Could not save credentials to file: %s\n", saveErr) + } + os.Exit(0) + } + gologger.Error().Msgf("Invalid API key '%v' got error: %v", maskKey(apiKey), err) + gologger.Fatal().Msgf("please recheck or recreate your API key and retry") +} + +func maskKey(key string) string { + if len(key) < 6 { + // this is invalid key + return key + } + return fmt.Sprintf("%v%v", key[:3], strings.Repeat("*", len(key)-3)) +} diff --git a/internal/pdcp/creds.go b/internal/pdcp/creds.go new file mode 100644 index 00000000..61cd2a3d --- /dev/null +++ b/internal/pdcp/creds.go @@ -0,0 +1,139 @@ +package pdcp + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/utils/env" + fileutil "github.com/projectdiscovery/utils/file" + folderutil "github.com/projectdiscovery/utils/folder" + urlutil "github.com/projectdiscovery/utils/url" + "gopkg.in/yaml.v3" +) + +var ( + PDCPDir = filepath.Join(folderutil.HomeDirOrDefault(""), ".pdcp") + PDCPCredFile = filepath.Join(PDCPDir, "credentials.yaml") + ErrNoCreds = fmt.Errorf("no credentials found in %s", PDCPDir) +) + +const ( + userProfileURL = "https://%s/v1/template/user/profile?utm_source=%s" + apiKeyEnv = "PDCP_API_KEY" + apiServerEnv = "PDCP_API_SERVER" + ApiKeyHeaderName = "X-Api-Key" + dashBoardEnv = "PDCP_DASHBOARD_URL" +) + +type PDCPCredentials struct { + Username string `yaml:"username"` + APIKey string `yaml:"api-key"` + Server string `yaml:"server"` +} + +type PDCPUserProfileResponse struct { + UserName string `json:"name"` + // there are more fields but we don't need them + /// below fields are added later on and not part of the response +} + +// PDCPCredHandler is interface for adding / retrieving pdcp credentials +// from file system +type PDCPCredHandler struct{} + +// GetCreds retrieves the credentials from the file system or environment variables +func (p *PDCPCredHandler) GetCreds() (*PDCPCredentials, error) { + credsFromEnv := p.getCredsFromEnv() + if credsFromEnv != nil { + return credsFromEnv, nil + } + if !fileutil.FolderExists(PDCPDir) || !fileutil.FileExists(PDCPCredFile) { + return nil, ErrNoCreds + } + bin, err := os.Open(PDCPCredFile) + if err != nil { + return nil, err + } + // for future use-cases + var creds []PDCPCredentials + err = yaml.NewDecoder(bin).Decode(&creds) + if err != nil { + return nil, err + } + if len(creds) == 0 { + return nil, ErrNoCreds + } + return &creds[0], nil +} + +// getCredsFromEnv retrieves the credentials from the environment +// if not or incomplete credentials are found it return nil +func (p *PDCPCredHandler) getCredsFromEnv() *PDCPCredentials { + apiKey := env.GetEnvOrDefault(apiKeyEnv, "") + apiServer := env.GetEnvOrDefault(apiServerEnv, "") + if apiKey == "" || apiServer == "" { + return nil + } + return &PDCPCredentials{APIKey: apiKey, Server: apiServer} +} + +// SaveCreds saves the credentials to the file system +func (p *PDCPCredHandler) SaveCreds(resp *PDCPCredentials) error { + if resp == nil { + return fmt.Errorf("invalid response") + } + if !fileutil.FolderExists(PDCPDir) { + _ = fileutil.CreateFolder(PDCPDir) + } + bin, err := yaml.Marshal([]*PDCPCredentials{resp}) + if err != nil { + return err + } + return os.WriteFile(PDCPCredFile, bin, 0600) +} + +// ValidateAPIKey validates the api key and retrieves associated user metadata like username +// from given api server/host +func (p *PDCPCredHandler) ValidateAPIKey(key string, host string, toolName string) (*PDCPCredentials, error) { + // get address from url + urlx, err := urlutil.Parse(host) + if err != nil { + return nil, err + } + req, err := retryablehttp.NewRequest("GET", fmt.Sprintf(userProfileURL, urlx.Host, toolName), nil) + if err != nil { + return nil, err + } + req.Header.Set(ApiKeyHeaderName, key) + resp, err := retryablehttp.DefaultHTTPClient.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != 200 { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + return nil, fmt.Errorf("invalid status code: %d", resp.StatusCode) + } + defer resp.Body.Close() + bin, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + var profile PDCPUserProfileResponse + err = json.Unmarshal(bin, &profile) + if err != nil { + return nil, err + } + if profile.UserName == "" { + return nil, fmt.Errorf("invalid response from server got %v", string(bin)) + } + return &PDCPCredentials{Username: profile.UserName, APIKey: key, Server: host}, nil +} + +func init() { + DashBoardURL = env.GetEnvOrDefault("PDCP_DASHBOARD_URL", DashBoardURL) +} diff --git a/internal/pdcp/creds_test.go b/internal/pdcp/creds_test.go new file mode 100644 index 00000000..bf800515 --- /dev/null +++ b/internal/pdcp/creds_test.go @@ -0,0 +1,33 @@ +package pdcp + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +var exampleCred = ` +- username: test + api-key: testpassword + server: https://scanme.sh +` + +func TestLoadCreds(t *testing.T) { + // temporarily change PDCP file location for testing + f, err := os.CreateTemp("", "creds-test-*") + require.Nil(t, err) + _, _ = f.WriteString(strings.TrimSpace(exampleCred)) + defer os.Remove(f.Name()) + PDCPCredFile = f.Name() + PDCPDir = filepath.Dir(f.Name()) + h := &PDCPCredHandler{} + value, err := h.GetCreds() + require.Nil(t, err) + require.NotNil(t, value) + require.Equal(t, "test", value.Username) + require.Equal(t, "testpassword", value.APIKey) + require.Equal(t, "https://scanme.sh", value.Server) +} diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go new file mode 100644 index 00000000..9d2e2a81 --- /dev/null +++ b/internal/pdcp/writer.go @@ -0,0 +1,157 @@ +package pdcp + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v3/pkg/output" + "github.com/projectdiscovery/retryablehttp-go" + errorutil "github.com/projectdiscovery/utils/errors" + fileutil "github.com/projectdiscovery/utils/file" + folderutil "github.com/projectdiscovery/utils/folder" + urlutil "github.com/projectdiscovery/utils/url" +) + +const ( + uploadEndpoint = "/v1/scans/import" +) + +var _ output.Writer = &UploadWriter{} + +// UploadWriter is a writer that uploads its output to pdcp +// server to enable web dashboard and more +type UploadWriter struct { + *output.StandardWriter + creds *PDCPCredentials + tempFile *os.File + done atomic.Bool + uploadURL *url.URL +} + +// NewUploadWriter creates a new upload writer +func NewUploadWriter(creds *PDCPCredentials) (*UploadWriter, error) { + if creds == nil { + return nil, fmt.Errorf("no credentials provided") + } + u := &UploadWriter{creds: creds} + // create a temporary file in cache directory + cacheDir := folderutil.AppCacheDirOrDefault("", config.BinaryName) + if !fileutil.FolderExists(cacheDir) { + _ = fileutil.CreateFolder(cacheDir) + } + + var err error + // tempfile is created in nuclei-results-.json format + u.tempFile, err = os.OpenFile(filepath.Join(cacheDir, "nuclei-results-"+strconv.Itoa(int(time.Now().Unix()))+".json"), os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not create temporary file") + } + u.StandardWriter, err = output.NewWriter( + output.WithWriter(u.tempFile), + output.WithJson(true, false), + ) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not create output writer") + } + tmp, err := urlutil.Parse(creds.Server) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not parse server url") + } + tmp.Path = uploadEndpoint + tmp.Update() + u.uploadURL = tmp.URL + return u, nil +} + +type uploadResponse struct { + ID string `json:"id"` + Message string `json:"message"` +} + +// Upload uploads the results to pdcp server +func (u *UploadWriter) Upload() { + defer u.done.Store(true) + + // start from beginning + _, _ = u.tempFile.Seek(0, 0) + // skip if file is empty + scanner := bufio.NewScanner(u.tempFile) + if !scanner.Scan() || (scanner.Scan() && strings.TrimSpace(scanner.Text()) == "") { + gologger.Verbose().Msgf("Scan results upload to cloud skipped, no results found to upload") + return + + } + _, _ = u.tempFile.Seek(0, 0) + + id, err := u.upload() + if err != nil { + gologger.Error().Msgf("Failed to upload scan results on cloud: %v", err) + return + } + gologger.Info().Msgf("Scan results uploaded! View them at %v", getScanDashBoardURL(id)) +} + +func (u *UploadWriter) upload() (string, error) { + req, err := retryablehttp.NewRequest(http.MethodPost, u.uploadURL.String(), u.tempFile) + if err != nil { + return "", errorutil.NewWithErr(err).Msgf("could not create cloud upload request") + } + req.Header.Set(ApiKeyHeaderName, u.creds.APIKey) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Accept", "application/json") + + opts := retryablehttp.DefaultOptionsSingle + // we are uploading nuclei results which can be large + // server has a size limit of ~20ish MB + opts.Timeout = time.Duration(3) * time.Minute + client := retryablehttp.NewClient(opts) + resp, err := client.Do(req) + if err != nil { + return "", errorutil.NewWithErr(err).Msgf("could not upload results") + } + defer resp.Body.Close() + bin, err := io.ReadAll(resp.Body) + if err != nil { + return "", errorutil.NewWithErr(err).Msgf("could not get id from response") + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("could not upload results got status code %v", resp.StatusCode) + } + var uploadResp uploadResponse + if err := json.Unmarshal(bin, &uploadResp); err != nil { + return "", errorutil.NewWithErr(err).Msgf("could not unmarshal response got %v", string(bin)) + } + u.removeTempFile() + return uploadResp.ID, nil +} + +// removeTempFile removes the temporary file +func (u *UploadWriter) removeTempFile() { + _ = os.Remove(u.tempFile.Name()) +} + +// Close closes the upload writer +func (u *UploadWriter) Close() { + if !u.done.Load() { + u.Upload() + } +} + +func getScanDashBoardURL(id string) string { + ux, _ := urlutil.Parse(DashBoardURL) + ux.Path = "/scans/" + id + ux.Update() + return ux.String() +} diff --git a/internal/runner/banner.go b/internal/runner/banner.go index d5157d98..3a5d7a05 100644 --- a/internal/runner/banner.go +++ b/internal/runner/banner.go @@ -1,9 +1,11 @@ +// Package runner executes the enumeration process. package runner import ( "fmt" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/internal/pdcp" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" updateutils "github.com/projectdiscovery/utils/update" ) @@ -25,5 +27,11 @@ func showBanner() { // NucleiToolUpdateCallback updates nuclei binary/tool to latest version func NucleiToolUpdateCallback() { showBanner() - updateutils.GetUpdateToolCallback("nuclei", config.Version)() + updateutils.GetUpdateToolCallback(config.BinaryName, config.Version)() +} + +// AuthWithPDCP is used to authenticate with PDCP +func AuthWithPDCP() { + showBanner() + pdcp.CheckNValidateCredentials(config.BinaryName) } diff --git a/internal/runner/cloud.go b/internal/runner/cloud.go deleted file mode 100644 index da58f15d..00000000 --- a/internal/runner/cloud.go +++ /dev/null @@ -1,438 +0,0 @@ -package runner - -import ( - "io" - "io/fs" - "os" - "path" - "path/filepath" - "strconv" - "strings" - - jsoniter "github.com/json-iterator/go" - "github.com/olekukonko/tablewriter" - "github.com/pkg/errors" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v3/internal/runner/nucleicloud" - "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/projectdiscovery/nuclei/v3/pkg/templates/extensions" -) - -// Get all the scan lists for a user/apikey. -func (r *Runner) getScanList(limit int) error { - lastTime := "2099-01-02 15:04:05 +0000 UTC" - header := []string{"ID", "Timestamp", "Targets", "Templates", "Matched", "Duration", "Status"} - - var ( - values [][]string - count int - ) - for { - items, err := r.cloudClient.GetScans(limit, lastTime) - if err != nil { - return err - } - if len(items) == 0 { - break - } - for _, v := range items { - count++ - lastTime = v.CreatedAt.String() - res := nucleicloud.PrepareScanListOutput(v) - if r.options.JSONL { - _ = jsoniter.NewEncoder(os.Stdout).Encode(res) - } else if !r.options.NoTables { - values = append(values, []string{strconv.FormatInt(res.ScanID, 10), res.Timestamp, strconv.Itoa(res.Target), strconv.Itoa(res.Template), strconv.Itoa(res.ScanResult), res.ScanTime, res.ScanStatus}) - } else { - gologger.Silent().Msgf("%d. [%s] [TARGETS: %d] [TEMPLATES: %d] [MATCHED: %d] [DURATION: %s] [STATUS: %s]\n", res.ScanID, res.Timestamp, res.Target, res.Template, res.ScanResult, res.ScanTime, strings.ToUpper(res.ScanStatus)) - - } - } - } - if count == 0 { - return errors.New("no scan found") - } - if !r.options.NoTables { - r.prettyPrintTable(header, values) - } - return nil -} - -func (r *Runner) listDatasources() error { - datasources, err := r.cloudClient.ListDatasources() - if err != nil { - return err - } - if len(datasources) == 0 { - return errors.New("no cloud datasource found") - } - - header := []string{"ID", "UpdatedAt", "Type", "Repo", "Path"} - var values [][]string - for _, source := range datasources { - if r.options.JSONL { - _ = jsoniter.NewEncoder(os.Stdout).Encode(source) - } else if !r.options.NoTables { - values = append(values, []string{strconv.FormatInt(source.ID, 10), source.Updatedat.Format(nucleicloud.DDMMYYYYhhmmss), source.Type, source.Repo, source.Path}) - } else { - gologger.Silent().Msgf("%d. [%s] [%s] [%s] %s", source.ID, source.Updatedat.Format(nucleicloud.DDMMYYYYhhmmss), source.Type, source.Repo, source.Path) - } - } - if !r.options.NoTables { - r.prettyPrintTable(header, values) - } - return nil -} - -func (r *Runner) listReportingSources() error { - items, err := r.cloudClient.ListReportingSources() - if err != nil { - return err - } - if len(items) == 0 { - return errors.New("no reporting source found") - } - - header := []string{"ID", "Type", "ProjectName", "Enabled"} - var values [][]string - for _, source := range items { - if r.options.JSONL { - _ = jsoniter.NewEncoder(os.Stdout).Encode(source) - } else if !r.options.NoTables { - values = append(values, []string{strconv.FormatInt(source.ID, 10), source.Type, source.ProjectName, strconv.FormatBool(source.Enabled)}) - } else { - gologger.Silent().Msgf("%d. [%s] [%s] [%t]", source.ID, source.Type, source.ProjectName, source.Enabled) - } - } - - if !r.options.NoTables { - r.prettyPrintTable(header, values) - } - return nil -} - -func (r *Runner) listTargets() error { - items, err := r.cloudClient.ListTargets("") - if err != nil { - return err - } - if len(items) == 0 { - return errors.New("no target found") - } - - header := []string{"ID", "Reference", "Count"} - var values [][]string - for _, source := range items { - if r.options.JSONL { - _ = jsoniter.NewEncoder(os.Stdout).Encode(source) - } else if !r.options.NoTables { - values = append(values, []string{strconv.FormatInt(source.ID, 10), source.Reference, strconv.FormatInt(source.Count, 10)}) - } else { - gologger.Silent().Msgf("%d. %s (%d)", source.ID, source.Reference, source.Count) - } - } - if !r.options.NoTables { - r.prettyPrintTable(header, values) - } - return nil -} - -func (r *Runner) listTemplates() error { - items, err := r.cloudClient.ListTemplates("") - if err != nil { - return err - } - if len(items) == 0 { - return errors.New("no template found") - } - - header := []string{"ID", "Reference"} - var values [][]string - for _, source := range items { - if r.options.JSONL { - _ = jsoniter.NewEncoder(os.Stdout).Encode(source) - } else if !r.options.NoTables { - values = append(values, []string{strconv.FormatInt(source.ID, 10), source.Reference}) - } else { - gologger.Silent().Msgf("%d. %s", source.ID, source.Reference) - } - } - if !r.options.NoTables { - r.prettyPrintTable(header, values) - } - return nil -} - -func (r *Runner) prettyPrintTable(header []string, values [][]string) { - writer := tablewriter.NewWriter(os.Stdout) - writer.SetHeader(header) - writer.AppendBulk(values) - writer.Render() -} - -func (r *Runner) deleteScan(id string) error { - ID, parseErr := strconv.ParseInt(id, 10, 64) - if parseErr != nil { - return errors.Wrap(parseErr, "could not parse scan id") - } - deleted, err := r.cloudClient.DeleteScan(ID) - if err != nil { - return errors.Wrap(err, "could not delete scan") - } - if !deleted.OK { - gologger.Error().Msgf("Error in deleting the scan %s.", id) - } else { - gologger.Info().Msgf("Scan deleted %s.", id) - } - return nil -} - -func (r *Runner) getResults(id string, limit int) error { - ID, _ := strconv.ParseInt(id, 10, 64) - err := r.cloudClient.GetResults(ID, false, limit, func(re *output.ResultEvent) { - if outputErr := r.output.Write(re); outputErr != nil { - gologger.Warning().Msgf("Could not write output: %s", outputErr) - } - }) - return err -} - -func (r *Runner) getTarget(id string) error { - var name string - ID, parseErr := strconv.ParseInt(id, 10, 64) - if parseErr != nil { - name = id - } - - reader, err := r.cloudClient.GetTarget(ID, name) - if err != nil { - return errors.Wrap(err, "could not get target") - } - defer reader.Close() - - _, _ = io.Copy(os.Stdout, reader) - return nil -} - -func (r *Runner) getTemplate(id string) error { - var name string - ID, parseErr := strconv.ParseInt(id, 10, 64) - if parseErr != nil { - name = id - } - - reader, err := r.cloudClient.GetTemplate(ID, name) - if err != nil { - return errors.Wrap(err, "could not get template") - } - defer reader.Close() - - _, _ = io.Copy(os.Stdout, reader) - return nil -} - -func (r *Runner) removeDatasource(datasource string) error { - var source string - ID, parseErr := strconv.ParseInt(datasource, 10, 64) - if parseErr != nil { - source = datasource - } - - err := r.cloudClient.RemoveDatasource(ID, source) - if err == nil { - gologger.Info().Msgf("Datasource deleted %s", datasource) - } - return err -} - -func (r *Runner) toggleReportingSource(source string, status bool) error { - ID, parseErr := strconv.ParseInt(source, 10, 64) - if parseErr != nil { - return errors.Wrap(parseErr, "could not parse reporting source id") - } - - err := r.cloudClient.ToggleReportingSource(ID, status) - if err == nil { - t := "enabled" - if !status { - t = "disabled" - } - gologger.Info().Msgf("Reporting source %s %s", t, source) - } - return err -} - -func (r *Runner) addTemplate(location string) error { - walkErr := filepath.WalkDir(location, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() || !strings.EqualFold(filepath.Ext(path), extensions.YAML) { - return nil - } - base := filepath.Base(path) - reference, templateErr := r.cloudClient.AddTemplate(base, path) - if templateErr != nil { - gologger.Error().Msgf("Could not upload %s: %s", path, templateErr) - } else if reference != "" { - gologger.Info().Msgf("Uploaded template %s: %s", base, reference) - } - return nil - }) - return walkErr -} - -func (r *Runner) addTarget(location string) error { - walkErr := filepath.WalkDir(location, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".txt") { - return nil - } - base := filepath.Base(path) - reference, targetErr := r.cloudClient.AddTarget(base, path) - if targetErr != nil { - gologger.Error().Msgf("Could not upload %s: %s", location, targetErr) - } else if reference != "" { - gologger.Info().Msgf("Uploaded target %s: %s", base, reference) - } - return nil - }) - return walkErr -} - -func (r *Runner) removeTarget(item string) error { - var err error - if ID, parseErr := strconv.ParseInt(item, 10, 64); parseErr == nil { - err = r.cloudClient.RemoveTarget(ID, "") - } else if strings.EqualFold(path.Ext(item), ".txt") { - err = r.cloudClient.RemoveTarget(0, item) - } else { - return r.removeTargetPrefix(item) - } - if err != nil { - gologger.Error().Msgf("Error in deleting target %s: %s", item, err) - } else { - gologger.Info().Msgf("Target deleted %s", item) - } - return nil -} - -func (r *Runner) removeTargetPrefix(item string) error { - response, err := r.cloudClient.ListTargets(item) - if err != nil { - return errors.Wrap(err, "could not list targets") - } - for _, item := range response { - if err := r.cloudClient.RemoveTarget(item.ID, ""); err != nil { - gologger.Error().Msgf("Error in deleting target %s: %s", item.Reference, err) - } else { - gologger.Info().Msgf("Target deleted %s", item.Reference) - } - } - return nil -} - -func (r *Runner) removeTemplate(item string) error { - var err error - if ID, parseErr := strconv.ParseInt(item, 10, 64); parseErr == nil { - err = r.cloudClient.RemoveTemplate(ID, "") - } else if strings.EqualFold(path.Ext(item), extensions.YAML) { - err = r.cloudClient.RemoveTemplate(0, item) - } else { - return r.removeTemplatePrefix(item) - } - if err != nil { - gologger.Error().Msgf("Error in deleting template %s: %s", item, err) - } else { - gologger.Info().Msgf("Template deleted %s", item) - } - return nil -} - -func (r *Runner) removeTemplatePrefix(item string) error { - response, err := r.cloudClient.ListTemplates(item) - if err != nil { - return errors.Wrap(err, "could not list templates") - } - for _, item := range response { - if err := r.cloudClient.RemoveTemplate(item.ID, ""); err != nil { - gologger.Error().Msgf("Error in deleting template %s: %s", item.Reference, err) - } else { - gologger.Info().Msgf("Template deleted %s", item.Reference) - } - } - return nil -} - -// initializeCloudDataSources initializes cloud data sources -func (r *Runner) addCloudDataSource(source string) error { - switch source { - case "s3": - token := strings.Join([]string{r.options.AwsAccessKey, r.options.AwsSecretKey, r.options.AwsRegion}, ":") - if _, err := r.processDataSourceItem(r.options.AwsBucketName, token, "s3"); err != nil { - return err - } - case "github": - for _, repo := range r.options.GitHubTemplateRepo { - if _, err := r.processDataSourceItem(repo, r.options.GitHubToken, "github"); err != nil { - return err - } - } - } - return nil -} - -func (r *Runner) processDataSourceItem(repo, token, Type string) (int64, error) { - ID, err := r.cloudClient.StatusDataSource(nucleicloud.StatusDataSourceRequest{Repo: repo, Token: token}) - if err != nil { - if !strings.Contains(err.Error(), "no rows in result set") { - return 0, errors.Wrap(err, "could not get data source status") - } - - gologger.Info().Msgf("Adding new data source + syncing: %s\n", repo) - resp, err := r.cloudClient.AddDataSource(nucleicloud.AddDataSourceRequest{Type: Type, Repo: repo, Token: token}) - if err != nil { - return 0, errors.Wrap(err, "could not add data source") - } - ID = resp.ID - if err = r.cloudClient.SyncDataSource(resp.ID); err != nil { - return 0, errors.Wrap(err, "could not sync data source") - } - if resp.Secret != "" { - gologger.Info().Msgf("Webhook URL for added source: %s/datasources/%s/webhook", r.options.CloudURL, resp.Hash) - gologger.Info().Msgf("Secret for webhook: %s", resp.Secret) - } - } - if r.options.UpdateTemplates { - gologger.Info().Msgf("Syncing data source: %s (%d)\n", repo, ID) - if err = r.cloudClient.SyncDataSource(ID); err != nil { - return 0, errors.Wrap(err, "could not sync data source") - } - } - return ID, nil -} - -// addCloudReportingSource adds reporting sources to cloud -func (r *Runner) addCloudReportingSource() error { - rcOptions := r.issuesClient.GetReportingOptions() - if rcOptions == nil { - return nil - } - if rcOptions.Jira != nil { - payload, err := jsoniter.Marshal(rcOptions.Jira) - if err != nil { - return err - } - requestObj := nucleicloud.AddReportingSourceRequest{ - Type: "jira", - Payload: payload, - } - if _, err := r.cloudClient.AddReportingSource(requestObj); err != nil { - return errors.Wrap(err, "could not add reporting source") - } - gologger.Info().Msgf("Reporting source and webhook added for %s: %s", "jira", r.options.CloudURL) - } - return nil -} diff --git a/internal/runner/defaults.go b/internal/runner/defaults.go deleted file mode 100644 index d51ab7c4..00000000 --- a/internal/runner/defaults.go +++ /dev/null @@ -1,6 +0,0 @@ -package runner - -const ( - // Default directory used to save protocols traffic - DefaultDumpTrafficOutputFolder = "output" -) diff --git a/internal/runner/doc.go b/internal/runner/doc.go deleted file mode 100644 index 6f0967cb..00000000 --- a/internal/runner/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package runner executes the enumeration process. -package runner diff --git a/internal/runner/enumerate.go b/internal/runner/enumerate.go deleted file mode 100644 index 31c079a4..00000000 --- a/internal/runner/enumerate.go +++ /dev/null @@ -1,166 +0,0 @@ -package runner - -import ( - "bytes" - "context" - "crypto/sha1" - "encoding/base64" - "encoding/hex" - "io" - _ "net/http/pprof" - "os" - "path/filepath" - "strings" - "sync/atomic" - "time" - - "github.com/klauspost/compress/zlib" - "github.com/pkg/errors" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v3/internal/runner/nucleicloud" - "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader" - "github.com/projectdiscovery/nuclei/v3/pkg/core" - "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v3/pkg/types" -) - -// runStandardEnumeration runs standard enumeration -func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecutorOptions, store *loader.Store, engine *core.Engine) (*atomic.Bool, error) { - if r.options.AutomaticScan { - return r.executeSmartWorkflowInput(executerOpts, store, engine) - } - return r.executeTemplatesInput(store, engine) -} - -// runCloudEnumeration runs cloud based enumeration -func (r *Runner) runCloudEnumeration(store *loader.Store, cloudTemplates, cloudTargets []string, nostore bool, limit int) (*atomic.Bool, error) { - count := &atomic.Int64{} - now := time.Now() - defer func() { - gologger.Info().Msgf("Scan execution took %s and found %d results", time.Since(now), count.Load()) - }() - results := &atomic.Bool{} - - // TODO: Add payload file and workflow support for private templates - catalogChecksums := nucleicloud.ReadCatalogChecksum() - - targets := make([]string, 0, r.hmapInputProvider.Count()) - r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool { - targets = append(targets, value.Input) - return true - }) - templates := make([]string, 0, len(store.Templates())) - privateTemplates := make(map[string]string) - - for _, template := range store.Templates() { - data, _ := os.ReadFile(template.Path) - h := sha1.New() - _, _ = io.Copy(h, bytes.NewReader(data)) - newhash := hex.EncodeToString(h.Sum(nil)) - - templateRelativePath := getTemplateRelativePath(template.Path) - if hash, ok := catalogChecksums[templateRelativePath]; ok || newhash == hash { - templates = append(templates, templateRelativePath) - } else { - privateTemplates[filepath.Base(template.Path)] = gzipBase64EncodeData(data) - } - } - - taskID, err := r.cloudClient.AddScan(&nucleicloud.AddScanRequest{ - RawTargets: targets, - PublicTemplates: templates, - CloudTargets: cloudTargets, - CloudTemplates: cloudTemplates, - PrivateTemplates: privateTemplates, - IsTemporary: nostore, - Filtering: getCloudFilteringFromOptions(r.options), - }) - if err != nil { - return results, err - } - gologger.Info().Msgf("Created task with ID: %d", taskID) - if nostore { - gologger.Info().Msgf("Cloud scan storage: disabled") - } - time.Sleep(3 * time.Second) - - scanResponse, err := r.cloudClient.GetScan(taskID) - if err != nil { - return results, errors.Wrap(err, "could not get scan status") - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // Start progress logging for the created scan - if r.progress != nil { - ticker := time.NewTicker(time.Duration(r.options.StatsInterval) * time.Second) - r.progress.Init(r.hmapInputProvider.Count(), int(scanResponse.Templates), int64(scanResponse.Total)) - go func() { - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - if scanResponse, err := r.cloudClient.GetScan(taskID); err == nil { - r.progress.SetRequests(uint64(scanResponse.Current)) - } - } - } - }() - } - - err = r.cloudClient.GetResults(taskID, true, limit, func(re *output.ResultEvent) { - r.progress.IncrementMatched() - results.CompareAndSwap(false, true) - _ = count.Add(1) - - if outputErr := r.output.Write(re); outputErr != nil { - gologger.Warning().Msgf("Could not write output: %s", err) - } - if r.issuesClient != nil { - if err := r.issuesClient.CreateIssue(re); err != nil { - gologger.Warning().Msgf("Could not create issue on tracker: %s", err) - } - } - }) - return results, err -} - -func getTemplateRelativePath(templatePath string) string { - splitted := strings.SplitN(templatePath, "nuclei-templates", 2) - if len(splitted) < 2 { - return "" - } - return strings.TrimPrefix(splitted[1], "/") -} - -func gzipBase64EncodeData(data []byte) string { - var buf bytes.Buffer - writer, _ := zlib.NewWriterLevel(&buf, zlib.BestCompression) - _, _ = writer.Write(data) - _ = writer.Close() - encoded := base64.StdEncoding.EncodeToString(buf.Bytes()) - return encoded -} - -func getCloudFilteringFromOptions(options *types.Options) *nucleicloud.AddScanRequestConfiguration { - return &nucleicloud.AddScanRequestConfiguration{ - Authors: options.Authors, - Tags: options.Tags, - ExcludeTags: options.ExcludeTags, - IncludeTags: options.IncludeTags, - IncludeIds: options.IncludeIds, - ExcludeIds: options.ExcludeIds, - IncludeTemplates: options.IncludeTemplates, - ExcludedTemplates: options.ExcludedTemplates, - ExcludeMatchers: options.ExcludeMatchers, - Severities: options.Severities, - ExcludeSeverities: options.ExcludeSeverities, - Protocols: options.Protocols, - ExcludeProtocols: options.ExcludeProtocols, - IncludeConditions: options.IncludeConditions, - } -} diff --git a/internal/runner/nucleicloud/cloud.go b/internal/runner/nucleicloud/cloud.go deleted file mode 100644 index 54b21640..00000000 --- a/internal/runner/nucleicloud/cloud.go +++ /dev/null @@ -1,666 +0,0 @@ -package nucleicloud - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "mime/multipart" - "net/http" - "net/url" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - jsoniter "github.com/json-iterator/go" - "github.com/pkg/errors" - "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/projectdiscovery/retryablehttp-go" -) - -// Client is a client for result retrieval from nuclei-cloud API -type Client struct { - baseURL string - apiKey string - httpclient *retryablehttp.Client -} - -const ( - pollInterval = 3 * time.Second - resultSize = 100 - defaultBaseURL = "https://cloud-dev.nuclei.sh" -) - -// HTTPErrorRetryPolicy is to retry for HTTPCodes >= 500. -func HTTPErrorRetryPolicy() func(ctx context.Context, resp *http.Response, err error) (bool, error) { - return func(ctx context.Context, resp *http.Response, err error) (bool, error) { - if resp != nil && resp.StatusCode >= http.StatusInternalServerError { - return true, errors.New(resp.Status) - } - return retryablehttp.CheckRecoverableErrors(ctx, resp, err) - } -} - -// New returns a nuclei-cloud API client -func New(baseURL, apiKey string) *Client { - options := retryablehttp.DefaultOptionsSingle - options.NoAdjustTimeout = true - options.Timeout = 60 * time.Second - options.CheckRetry = HTTPErrorRetryPolicy() - client := retryablehttp.NewClient(options) - - baseAppURL := baseURL - if baseAppURL == "" { - baseAppURL = defaultBaseURL - } - return &Client{httpclient: client, baseURL: baseAppURL, apiKey: apiKey} -} - -// AddScan adds a scan for templates and target to nuclei server -func (c *Client) AddScan(req *AddScanRequest) (int64, error) { - var buf bytes.Buffer - if err := jsoniter.NewEncoder(&buf).Encode(req); err != nil { - return 0, errors.Wrap(err, "could not encode request") - } - httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/scan", c.baseURL), bytes.NewReader(buf.Bytes())) - if err != nil { - return 0, errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return 0, errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - - var data map[string]int64 - if err := jsoniter.NewDecoder(resp.Body).Decode(&data); err != nil { - return 0, errors.Wrap(err, "could not decode resp") - } - id := data["id"] - return id, nil -} - -// GetResults gets results from nuclei server for an ID -// until there are no more results left to retrieve. -func (c *Client) GetResults(ID int64, checkProgress bool, limit int, callback func(*output.ResultEvent)) error { - lastID := int64(0) - - for { - uri := fmt.Sprintf("%s/results?id=%d&from=%d&size=%d", c.baseURL, ID, lastID, limit) - httpReq, err := retryablehttp.NewRequest(http.MethodGet, uri, nil) - if err != nil { - return errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return errors.Wrap(err, "could not do request") - } - - var items GetResultsResponse - if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil { - resp.Body.Close() - return errors.Wrap(err, "could not decode results") - } - resp.Body.Close() - - for _, item := range items.Items { - lastID = item.ID - - var result output.ResultEvent - if err := jsoniter.NewDecoder(strings.NewReader(item.Raw)).Decode(&result); err != nil { - return errors.Wrap(err, "could not decode result item") - } - callback(&result) - } - - // This is checked during scan is added else if no item found break out of loop. - if checkProgress { - if items.Finished && len(items.Items) == 0 { - break - } - } else if len(items.Items) == 0 { - break - } - - time.Sleep(pollInterval) - } - return nil -} - -func (c *Client) GetScans(limit int, from string) ([]GetScanRequest, error) { - var items []GetScanRequest - httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/scan?from=%s&size=%d", c.baseURL, url.QueryEscape(from), limit), nil) - if err != nil { - return items, errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return nil, errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - - if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil { - return items, errors.Wrap(err, "could not decode results") - } - return items, nil -} - -func (c *Client) GetScan(id int64) (GetScanRequest, error) { - var items GetScanRequest - httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/scan/%d", c.baseURL, id), nil) - if err != nil { - return items, errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return items, errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - - if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil { - return items, errors.Wrap(err, "could not decode results") - } - return items, nil -} - -// Delete a scan and it's issues by the scan id. -func (c *Client) DeleteScan(id int64) (DeleteScanResults, error) { - deletescan := DeleteScanResults{} - httpReq, err := retryablehttp.NewRequest(http.MethodDelete, fmt.Sprintf("%s/scan?id=%d", c.baseURL, id), nil) - if err != nil { - return deletescan, errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return deletescan, errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - - if err := jsoniter.NewDecoder(resp.Body).Decode(&deletescan); err != nil { - return deletescan, errors.Wrap(err, "could not delete scan") - } - return deletescan, nil -} - -// StatusDataSource returns the status for a data source -func (c *Client) StatusDataSource(statusRequest StatusDataSourceRequest) (int64, error) { - var buf bytes.Buffer - if err := jsoniter.NewEncoder(&buf).Encode(statusRequest); err != nil { - return 0, errors.Wrap(err, "could not encode request") - } - httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/datasources/status", c.baseURL), bytes.NewReader(buf.Bytes())) - if err != nil { - return 0, errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return 0, errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - - var data StatusDataSourceResponse - if err := jsoniter.NewDecoder(resp.Body).Decode(&data); err != nil { - return 0, errors.Wrap(err, "could not decode resp") - } - return data.ID, nil -} - -// AddDataSource adds a new data source -func (c *Client) AddDataSource(req AddDataSourceRequest) (*AddDataSourceResponse, error) { - var buf bytes.Buffer - if err := jsoniter.NewEncoder(&buf).Encode(req); err != nil { - return nil, errors.Wrap(err, "could not encode request") - } - httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/datasources", c.baseURL), bytes.NewReader(buf.Bytes())) - if err != nil { - return nil, errors.Wrap(err, "could not make request") - } - resp, err := c.sendRequest(httpReq) - if err != nil { - return nil, errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - - var data AddDataSourceResponse - if err := jsoniter.NewDecoder(resp.Body).Decode(&data); err != nil { - return nil, errors.Wrap(err, "could not decode resp") - } - return &data, nil -} - -// SyncDataSource syncs contents for a data source. The call blocks until -// update is completed. -func (c *Client) SyncDataSource(ID int64) error { - httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/datasources/%d/sync", c.baseURL, ID), nil) - if err != nil { - return errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - _, _ = io.Copy(io.Discard, resp.Body) - return nil -} - -// ExistsDataSourceItem identifies whether data source item exist -func (c *Client) ExistsDataSourceItem(req ExistsDataSourceItemRequest) error { - var buf bytes.Buffer - if err := jsoniter.NewEncoder(&buf).Encode(req); err != nil { - return errors.Wrap(err, "could not encode request") - } - httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/datasources/exists", c.baseURL), bytes.NewReader(buf.Bytes())) - if err != nil { - return errors.Wrap(err, "could not make request") - } - resp, err := c.sendRequest(httpReq) - if err != nil { - return errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - _, _ = io.Copy(io.Discard, resp.Body) - return nil -} - -func (c *Client) ListDatasources() ([]GetDataSourceResponse, error) { - var items []GetDataSourceResponse - httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/datasources", c.baseURL), nil) - if err != nil { - return items, errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return nil, errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - - if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil { - return items, errors.Wrap(err, "could not decode results") - } - return items, nil -} - -func (c *Client) ListReportingSources() ([]GetReportingSourceResponse, error) { - var items []GetReportingSourceResponse - httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/reporting", c.baseURL), nil) - if err != nil { - return items, errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return nil, errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - - if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil { - return items, errors.Wrap(err, "could not decode results") - } - return items, nil -} - -func (c *Client) ToggleReportingSource(ID int64, status bool) error { - r := ReportingSourceStatus{Enabled: status} - - var buf bytes.Buffer - if err := jsoniter.NewEncoder(&buf).Encode(r); err != nil { - return errors.Wrap(err, "could not encode request") - } - httpReq, err := retryablehttp.NewRequest(http.MethodPut, fmt.Sprintf("%s/reporting/%d", c.baseURL, ID), bytes.NewReader(buf.Bytes())) - if err != nil { - return errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - _, _ = io.Copy(io.Discard, resp.Body) - return nil -} - -func (c *Client) ListTargets(query string) ([]GetTargetResponse, error) { - var builder strings.Builder - _, _ = builder.WriteString(c.baseURL) - _, _ = builder.WriteString("/targets") - if query != "" { - _, _ = builder.WriteString("?query=") - _, _ = builder.WriteString(url.QueryEscape(query)) - } - - var items []GetTargetResponse - httpReq, err := retryablehttp.NewRequest(http.MethodGet, builder.String(), nil) - if err != nil { - return items, errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return nil, errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - - if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil { - return items, errors.Wrap(err, "could not decode results") - } - return items, nil -} - -func (c *Client) ListTemplates(query string) ([]GetTemplatesResponse, error) { - var builder strings.Builder - _, _ = builder.WriteString(c.baseURL) - _, _ = builder.WriteString("/templates") - if query != "" { - _, _ = builder.WriteString("?query=") - _, _ = builder.WriteString(url.QueryEscape(query)) - } - - var items []GetTemplatesResponse - httpReq, err := retryablehttp.NewRequest(http.MethodGet, builder.String(), nil) - if err != nil { - return items, errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return nil, errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - - if err := jsoniter.NewDecoder(resp.Body).Decode(&items); err != nil { - return items, errors.Wrap(err, "could not decode results") - } - return items, nil -} - -func (c *Client) RemoveDatasource(datasource int64, name string) error { - var builder strings.Builder - _, _ = builder.WriteString(c.baseURL) - _, _ = builder.WriteString("/datasources") - - if name != "" { - _, _ = builder.WriteString("?name=") - _, _ = builder.WriteString(name) - } else if datasource != 0 { - _, _ = builder.WriteString("?id=") - _, _ = builder.WriteString(strconv.FormatInt(datasource, 10)) - } - - httpReq, err := retryablehttp.NewRequest(http.MethodDelete, builder.String(), nil) - if err != nil { - return errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - _, _ = io.Copy(io.Discard, resp.Body) - return nil -} - -func (c *Client) AddTemplate(name, contents string) (string, error) { - file, err := os.Open(contents) - if err != nil { - return "", errors.Wrap(err, "could not open contents") - } - defer file.Close() - - var buf bytes.Buffer - writer := multipart.NewWriter(&buf) - _ = writer.WriteField("name", name) - fileWriter, _ := writer.CreateFormFile("file", filepath.Base(contents)) - _, _ = io.Copy(fileWriter, file) - _ = writer.Close() - - httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/templates", c.baseURL), &buf) - if err != nil { - return "", errors.Wrap(err, "could not make request") - } - httpReq.Header.Set("Content-Type", writer.FormDataContentType()) - - resp, err := c.sendRequest(httpReq) - if err != nil { - return "", errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - - var item AddItemResponse - if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil { - return "", errors.Wrap(err, "could not decode results") - } - return item.Ok, nil -} - -func (c *Client) AddTarget(name, contents string) (string, error) { - file, err := os.Open(contents) - if err != nil { - return "", errors.Wrap(err, "could not open contents") - } - defer file.Close() - - var buf bytes.Buffer - writer := multipart.NewWriter(&buf) - _ = writer.WriteField("name", name) - fileWriter, _ := writer.CreateFormFile("file", filepath.Base(contents)) - _, _ = io.Copy(fileWriter, file) - _ = writer.Close() - - httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/targets", c.baseURL), &buf) - if err != nil { - return "", errors.Wrap(err, "could not make request") - } - httpReq.Header.Set("Content-Type", writer.FormDataContentType()) - - resp, err := c.sendRequest(httpReq) - if err != nil { - return "", errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - - var item AddItemResponse - if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil { - return "", errors.Wrap(err, "could not decode results") - } - return item.Ok, nil -} - -func (c *Client) RemoveTemplate(ID int64, name string) error { - var builder strings.Builder - _, _ = builder.WriteString(c.baseURL) - _, _ = builder.WriteString("/templates") - - if name != "" { - _, _ = builder.WriteString("?name=") - _, _ = builder.WriteString(name) - } else if ID != 0 { - _, _ = builder.WriteString("?id=") - _, _ = builder.WriteString(strconv.FormatInt(ID, 10)) - } - httpReq, err := retryablehttp.NewRequest(http.MethodDelete, builder.String(), nil) - if err != nil { - return errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - _, _ = io.Copy(io.Discard, resp.Body) - return nil -} - -func (c *Client) RemoveTarget(ID int64, name string) error { - var builder strings.Builder - _, _ = builder.WriteString(c.baseURL) - _, _ = builder.WriteString("/targets") - - if name != "" { - _, _ = builder.WriteString("?name=") - _, _ = builder.WriteString(name) - } else if ID != 0 { - _, _ = builder.WriteString("?id=") - _, _ = builder.WriteString(strconv.FormatInt(ID, 10)) - } - httpReq, err := retryablehttp.NewRequest(http.MethodDelete, builder.String(), nil) - if err != nil { - return errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - _, _ = io.Copy(io.Discard, resp.Body) - return nil -} - -func (c *Client) GetTarget(ID int64, name string) (io.ReadCloser, error) { - var builder strings.Builder - _, _ = builder.WriteString(c.baseURL) - _, _ = builder.WriteString("/targets/get") - - if name != "" { - _, _ = builder.WriteString("?name=") - _, _ = builder.WriteString(name) - } else if ID != 0 { - _, _ = builder.WriteString("?id=") - _, _ = builder.WriteString(strconv.FormatInt(ID, 10)) - } - httpReq, err := retryablehttp.NewRequest(http.MethodGet, builder.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return nil, errors.Wrap(err, "could not do request") - } - return resp.Body, nil -} - -func (c *Client) GetTemplate(ID int64, name string) (io.ReadCloser, error) { - var builder strings.Builder - _, _ = builder.WriteString(c.baseURL) - _, _ = builder.WriteString("/templates/get") - - if name != "" { - _, _ = builder.WriteString("?name=") - _, _ = builder.WriteString(name) - } else if ID != 0 { - _, _ = builder.WriteString("?id=") - _, _ = builder.WriteString(strconv.FormatInt(ID, 10)) - } - httpReq, err := retryablehttp.NewRequest(http.MethodGet, builder.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return nil, errors.Wrap(err, "could not do request") - } - return resp.Body, nil -} - -func (c *Client) ExistsTarget(id int64) (ExistsInputResponse, error) { - var item ExistsInputResponse - httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/targets/%d/exists", c.baseURL, id), nil) - if err != nil { - return item, errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return item, errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - - if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil { - return item, errors.Wrap(err, "could not decode results") - } - return item, nil -} - -func (c *Client) ExistsTemplate(id int64) (ExistsInputResponse, error) { - var item ExistsInputResponse - httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/templates/%d/exists", c.baseURL, id), nil) - if err != nil { - return item, errors.Wrap(err, "could not make request") - } - - resp, err := c.sendRequest(httpReq) - if err != nil { - return item, errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - - if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil { - return item, errors.Wrap(err, "could not decode results") - } - return item, nil -} - -const apiKeyParameter = "X-API-Key" - -type errorResponse struct { - Message string `json:"message"` -} - -func (c *Client) sendRequest(req *retryablehttp.Request) (*http.Response, error) { - req.Header.Set(apiKeyParameter, c.apiKey) - - resp, err := c.httpclient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "could not do request") - } - if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest { - data, _ := io.ReadAll(resp.Body) - resp.Body.Close() - var errRes errorResponse - if err = json.NewDecoder(bytes.NewReader(data)).Decode(&errRes); err == nil { - return nil, errors.New(errRes.Message) - } - return nil, fmt.Errorf("unknown error, status code: %d=%s", resp.StatusCode, string(data)) - } - return resp, nil -} - -// AddReportingSource adds a new data source -func (c *Client) AddReportingSource(req AddReportingSourceRequest) (*AddReportingSourceResponse, error) { - var buf bytes.Buffer - if err := jsoniter.NewEncoder(&buf).Encode(req); err != nil { - return nil, errors.Wrap(err, "could not encode request") - } - httpReq, err := retryablehttp.NewRequest(http.MethodPost, fmt.Sprintf("%s/reporting/add-source", c.baseURL), bytes.NewReader(buf.Bytes())) - if err != nil { - return nil, errors.Wrap(err, "could not make request") - } - resp, err := c.sendRequest(httpReq) - if err != nil { - return nil, errors.Wrap(err, "could not do request") - } - defer resp.Body.Close() - - var data AddReportingSourceResponse - if err := jsoniter.NewDecoder(resp.Body).Decode(&data); err != nil { - return nil, errors.Wrap(err, "could not decode resp") - } - return &data, nil -} diff --git a/internal/runner/nucleicloud/types.go b/internal/runner/nucleicloud/types.go deleted file mode 100644 index 6eaa7237..00000000 --- a/internal/runner/nucleicloud/types.go +++ /dev/null @@ -1,178 +0,0 @@ -package nucleicloud - -import ( - "encoding/json" - "time" - - "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" - "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" -) - -// AddScanRequest is a nuclei scan input item. -type AddScanRequest struct { - // RawTargets is a list of raw target URLs for the scan. - RawTargets []string `json:"raw_targets,omitempty"` - // PublicTemplates is a list of public templates for the scan - PublicTemplates []string `json:"public_templates,omitempty"` - // PrivateTemplates is a map of template-name->contents that - // are private to the user executing the scan. (TODO: TBD) - PrivateTemplates map[string]string `json:"private_templates,omitempty"` - // CloudTargets is a list of cloud targets for the scan - CloudTargets []string `json:"cloud_targets,omitempty"` - // CloudTemplates is a list of cloud templates for the scan - CloudTemplates []string `json:"cloud_templates,omitempty"` - // Filtering contains optional filtering options for scan additions - Filtering *AddScanRequestConfiguration `json:"filtering"` - - IsTemporary bool `json:"is_temporary"` -} - -// AddScanRequestConfiguration contains filtering options for scan addition -type AddScanRequestConfiguration struct { - Authors []string `json:"author,omitempty"` - Tags []string `json:"tags,omitempty"` - ExcludeTags []string `json:"exclude-tags,omitempty"` - IncludeTags []string `json:"include-tags,omitempty"` - IncludeIds []string `json:"include-ids,omitempty"` - ExcludeIds []string `json:"exclude-ids,omitempty"` - IncludeTemplates []string `json:"include-templates,omitempty"` - ExcludedTemplates []string `json:"exclude-templates,omitempty"` - ExcludeMatchers []string `json:"exclude-matchers,omitempty"` - Severities severity.Severities `json:"severities,omitempty"` - ExcludeSeverities severity.Severities `json:"exclude-severities,omitempty"` - Protocols types.ProtocolTypes `json:"protocols,omitempty"` - ExcludeProtocols types.ProtocolTypes `json:"exclude-protocols,omitempty"` - IncludeConditions []string `json:"include-conditions,omitempty"` -} - -type GetResultsResponse struct { - Finished bool `json:"finished"` - Items []GetResultsResponseItem `json:"items"` -} - -type GetScanRequest struct { - Id int64 `json:"id"` - Total int32 `json:"total"` - Current int32 `json:"current"` - Finished bool `json:"finished"` - CreatedAt time.Time `json:"created_at"` - FinishedAt time.Time `json:"finished_at"` - Targets int32 `json:"targets"` - Templates int32 `json:"templates"` - Matches int64 `json:"matches"` -} - -// AddDataSourceResponse is a add data source response item. -type AddDataSourceResponse struct { - ID int64 `json:"id"` - Hash string `json:"hash"` - Secret string `json:"secret,omitempty"` -} - -type GetResultsResponseItem struct { - ID int64 `json:"id"` - Raw string `json:"raw"` -} - -type DeleteScanResults struct { - OK bool `json:"ok"` -} - -// StatusDataSourceRequest is a add data source request item. -type StatusDataSourceRequest struct { - Repo string `json:"repo"` - Token string `json:"token"` -} - -// StatusDataSourceResponse is a add data source response item. -type StatusDataSourceResponse struct { - ID int64 `json:"id"` -} - -// AddDataSourceRequest is a add data source request item. -type AddDataSourceRequest struct { - Type string `json:"type"` - Repo string `json:"repo"` - Token string `json:"token"` - Sync bool `json:"sync"` -} - -// ExistsDataSourceItemRequest is a request to identify whether a data -// source item exists. -type ExistsDataSourceItemRequest struct { - Type string `json:"type"` - Contents string `json:"contents"` -} - -// GetDataSourceResponse is response for a get data source request -type GetDataSourceResponse struct { - ID int64 `json:"id"` - Hash string `json:"hash"` - Type string `json:"type"` - Path string `json:"path"` - Repo string `json:"repo"` - Updatedat time.Time `json:"updated_at"` -} - -// GetTargetResponse is the response for a get target request -type GetTargetResponse struct { - ID int64 `json:"id"` - DataSource int64 `json:"data_source"` - Name string `json:"name"` - Reference string `json:"reference"` - Count int64 `json:"count"` - Hash string `json:"hash"` - Type string `json:"type"` -} - -// GetTemplatesResponse is the response for a get templates request -type GetTemplatesResponse struct { - ID int64 `json:"id,omitempty"` - DataSource int64 `json:"data_source,omitempty"` - Name string `json:"name,omitempty"` - Reference string `json:"reference,omitempty"` - Hash string `json:"hash,omitempty"` - Type string `json:"type,omitempty"` -} - -type GetReportingSourceResponse struct { - ID int64 `json:"id"` - Type string `json:"type"` - ProjectName string `json:"project_name"` - Enabled bool `json:"enabled"` - Updatedat time.Time `json:"updated_at"` -} - -type ReportingSourceStatus struct { - Enabled bool `json:"enabled"` -} - -// AddItemResponse is the response to add item request -type AddItemResponse struct { - Ok string `json:"ok"` -} - -type ListScanOutput struct { - Timestamp string `json:"timestamp"` - ScanID int64 `json:"scan_id"` - ScanTime string `json:"scan_time"` - ScanResult int `json:"scan_result"` - ScanStatus string `json:"scan_status"` - Target int `json:"target"` - Template int `json:"template"` -} - -type ExistsInputResponse struct { - Reference string `json:"reference"` -} - -// AddReportingSourceRequest is a add reporting source request item. -type AddReportingSourceRequest struct { - Type string `json:"type"` - Payload json.RawMessage `json:"payload"` -} - -// AddReportingSourceResponse is a add reporting source response item. -type AddReportingSourceResponse struct { - Ok string `json:"ok"` -} diff --git a/internal/runner/nucleicloud/utils.go b/internal/runner/nucleicloud/utils.go deleted file mode 100644 index 628d9715..00000000 --- a/internal/runner/nucleicloud/utils.go +++ /dev/null @@ -1,66 +0,0 @@ -package nucleicloud - -import ( - "bufio" - "os" - "path/filepath" - "strings" - "time" - - "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" -) - -const DDMMYYYYhhmmss = "2006-01-02 15:04:05" - -// ReadCatalogChecksum reads catalog checksum from nuclei-templates repository -func ReadCatalogChecksum() map[string]string { - config := config.DefaultConfig - - checksumFile := filepath.Join(config.TemplatesDirectory, "templates-checksum.txt") - file, err := os.Open(checksumFile) - if err != nil { - return nil - } - defer file.Close() - - checksums := make(map[string]string) - scanner := bufio.NewScanner(file) - for scanner.Scan() { - text := strings.SplitN(scanner.Text(), ":", 2) - if len(text) < 2 { - continue - } - path := strings.TrimPrefix(text[0], "nuclei-templates/") - if strings.HasPrefix(path, ".") { - continue - } - checksums[path] = text[1] - } - return checksums -} - -func PrepareScanListOutput(v GetScanRequest) ListScanOutput { - output := ListScanOutput{} - loc, _ := time.LoadLocation("Local") - status := "finished" - - t := v.FinishedAt - duration := t.Sub(v.CreatedAt) - - if !v.Finished { - status = "running" - t = time.Now().UTC() - duration = t.Sub(v.CreatedAt).Round(60 * time.Second) - } - - val := v.CreatedAt.In(loc).Format(DDMMYYYYhhmmss) - - output.Timestamp = val - output.ScanID = v.Id - output.ScanTime = duration.String() - output.ScanResult = int(v.Matches) - output.ScanStatus = status - output.Target = int(v.Targets) - output.Template = int(v.Templates) - return output -} diff --git a/internal/runner/options.go b/internal/runner/options.go index 94bc4c36..828957c3 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -20,14 +20,24 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" - protocoltypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml" fileutil "github.com/projectdiscovery/utils/file" "github.com/projectdiscovery/utils/generic" logutil "github.com/projectdiscovery/utils/log" stringsutil "github.com/projectdiscovery/utils/strings" ) +const ( + // Default directory used to save protocols traffic + DefaultDumpTrafficOutputFolder = "output" +) + func ConfigureOptions() error { // with FileStringSliceOptions, FileNormalizedStringSliceOptions, FileCommaSeparatedStringSliceOptions // if file has the extension `.yaml` or `.json` we consider those as strings and not files to be read @@ -184,37 +194,6 @@ func ValidateOptions(options *types.Options) error { if !useIPV4 && !useIPV6 { return errors.New("ipv4 and/or ipv6 must be selected") } - - // Validate cloud option - if err := validateCloudOptions(options); err != nil { - return err - } - return nil -} - -func validateCloudOptions(options *types.Options) error { - if options.HasCloudOptions() && !options.Cloud { - return errors.New("cloud flags cannot be used without cloud option") - } - if options.Cloud { - if options.CloudAPIKey == "" { - return errors.New("missing NUCLEI_CLOUD_API env variable") - } - var missing []string - switch options.AddDatasource { - case "s3": - missing = validateMissingS3Options(options) - case "github": - missing = validateMissingGitHubOptions(options) - case "gitlab": - missing = validateMissingGitLabOptions(options) - case "azure": - missing = validateMissingAzureOptions(options) - } - if len(missing) > 0 { - return fmt.Errorf("missing %v env variables", strings.Join(missing, ", ")) - } - } return nil } @@ -255,17 +234,6 @@ func validateMissingAzureOptions(options *types.Options) []string { return missing } -func validateMissingGitHubOptions(options *types.Options) []string { - var missing []string - if options.GitHubToken == "" { - missing = append(missing, "GITHUB_TOKEN") - } - if len(options.GitHubTemplateRepo) == 0 { - missing = append(missing, "GITHUB_TEMPLATE_REPO") - } - return missing -} - func validateMissingGitLabOptions(options *types.Options) []string { var missing []string if options.GitLabToken == "" { @@ -278,6 +246,46 @@ func validateMissingGitLabOptions(options *types.Options) []string { return missing } +func createReportingOptions(options *types.Options) (*reporting.Options, error) { + var reportingOptions = &reporting.Options{} + if options.ReportingConfig != "" { + file, err := os.Open(options.ReportingConfig) + if err != nil { + return nil, errors.Wrap(err, "could not open reporting config file") + } + defer file.Close() + + if err := yaml.DecodeAndValidate(file, reportingOptions); err != nil { + return nil, errors.Wrap(err, "could not parse reporting config file") + } + Walk(reportingOptions, expandEndVars) + } + if options.MarkdownExportDirectory != "" { + reportingOptions.MarkdownExporter = &markdown.Options{ + Directory: options.MarkdownExportDirectory, + IncludeRawPayload: !options.OmitRawRequests, + SortMode: options.MarkdownExportSortMode, + } + } + if options.SarifExport != "" { + reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport} + } + if options.JSONExport != "" { + reportingOptions.JSONExporter = &jsonexporter.Options{ + File: options.JSONExport, + IncludeRawPayload: !options.OmitRawRequests, + } + } + if options.JSONLExport != "" { + reportingOptions.JSONLExporter = &jsonl.Options{ + File: options.JSONLExport, + IncludeRawPayload: !options.OmitRawRequests, + } + } + + return reportingOptions, nil +} + // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output @@ -354,17 +362,6 @@ func validateCertificatePaths(certificatePaths ...string) { // Read the input from env and set options func readEnvInputVars(options *types.Options) { - if strings.EqualFold(os.Getenv("NUCLEI_CLOUD"), "true") { - options.Cloud = true - - // TODO: disable files, offlinehttp, code - options.ExcludeProtocols = append(options.ExcludeProtocols, protocoltypes.CodeProtocol, protocoltypes.FileProtocol, protocoltypes.OfflineHTTPProtocol) - } - if options.CloudURL = os.Getenv("NUCLEI_CLOUD_SERVER"); options.CloudURL == "" { - options.CloudURL = "https://cloud-dev.nuclei.sh" - } - options.CloudAPIKey = os.Getenv("NUCLEI_CLOUD_API") - options.GitHubToken = os.Getenv("GITHUB_TOKEN") repolist := os.Getenv("GITHUB_TEMPLATE_REPO") if repolist != "" { diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 721a0a31..c78fb83e 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -3,18 +3,19 @@ package runner import ( "context" "encoding/json" + "fmt" "net/http" _ "net/http/pprof" "os" "reflect" - "strconv" "strings" "sync/atomic" "time" - "github.com/projectdiscovery/nuclei/v3/internal/runner/nucleicloud" + "github.com/projectdiscovery/nuclei/v3/internal/pdcp" "github.com/projectdiscovery/nuclei/v3/pkg/installer" uncoverlib "github.com/projectdiscovery/uncover" + "github.com/projectdiscovery/utils/env" permissionutil "github.com/projectdiscovery/utils/permission" updateutils "github.com/projectdiscovery/utils/update" @@ -47,10 +48,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" - "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter" - "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl" - "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown" - "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v3/pkg/templates" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/nuclei/v3/pkg/utils" @@ -60,6 +57,13 @@ import ( ptrutil "github.com/projectdiscovery/utils/ptr" ) +var ( + // HideAutoSaveMsg is a global variable to hide the auto-save message + HideAutoSaveMsg = false + // DisableCloudUpload is a global variable to disable cloud upload + DisableCloudUpload = false +) + // Runner is a client for running the enumeration process. type Runner struct { output output.Writer @@ -76,8 +80,8 @@ type Runner struct { hostErrors hosterrorscache.CacheInterface resumeCfg *types.ResumeCfg pprofServer *http.Server - cloudClient *nucleicloud.Client - cloudTargets []string + // pdcp auto-save options + pdcpUploadErrMsg string } const pprofServerAddress = "127.0.0.1:8086" @@ -93,10 +97,6 @@ func New(options *types.Options) (*Runner, error) { os.Exit(0) } - if options.Cloud { - runner.cloudClient = nucleicloud.New(options.CloudURL, options.CloudAPIKey) - } - // Version check by default if config.DefaultConfig.CanCheckForUpdates() { if err := installer.NucleiVersionCheck(); err != nil { @@ -210,31 +210,13 @@ func New(options *types.Options) (*Runner, error) { }() } - if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && (options.UpdateTemplates && !options.Cloud) { + if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates { os.Exit(0) } // Initialize the input source hmapInput, err := hybrid.New(&hybrid.Options{ Options: options, - NotFoundCallback: func(target string) bool { - if !options.Cloud { - return false - } - parsed, parseErr := strconv.ParseInt(target, 10, 64) - if parseErr != nil { - if err := runner.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Contents: target, Type: "targets"}); err == nil { - runner.cloudTargets = append(runner.cloudTargets, target) - return true - } - return false - } - if exists, err := runner.cloudClient.ExistsTarget(parsed); err == nil { - runner.cloudTargets = append(runner.cloudTargets, exists.Reference) - return true - } - return false - }, }) if err != nil { return nil, errors.Wrap(err, "could not create input provider") @@ -246,7 +228,8 @@ func New(options *types.Options) (*Runner, error) { if err != nil { return nil, errors.Wrap(err, "could not create output file") } - runner.output = outputWriter + // setup a proxy writer to automatically upload results to PDCP + runner.output = runner.setupPDCPUpload(outputWriter) if options.JSONL && options.EnableProgressBar { options.StatsJSON = true @@ -257,11 +240,7 @@ func New(options *types.Options) (*Runner, error) { // Creates the progress tracking object var progressErr error statsInterval := options.StatsInterval - if options.Cloud && !options.EnableProgressBar { - statsInterval = -1 - options.EnableProgressBar = true - } - runner.progress, progressErr = progress.NewStatsTicker(statsInterval, options.EnableProgressBar, options.StatsJSON, options.Cloud, options.MetricsPort) + runner.progress, progressErr = progress.NewStatsTicker(statsInterval, options.EnableProgressBar, options.StatsJSON, false, options.MetricsPort) if progressErr != nil { return nil, progressErr } @@ -336,44 +315,12 @@ func New(options *types.Options) (*Runner, error) { return runner, nil } -func createReportingOptions(options *types.Options) (*reporting.Options, error) { - var reportingOptions = &reporting.Options{} - if options.ReportingConfig != "" { - file, err := os.Open(options.ReportingConfig) - if err != nil { - return nil, errors.Wrap(err, "could not open reporting config file") - } - defer file.Close() - - if err := yaml.DecodeAndValidate(file, reportingOptions); err != nil { - return nil, errors.Wrap(err, "could not parse reporting config file") - } - Walk(reportingOptions, expandEndVars) +// runStandardEnumeration runs standard enumeration +func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecutorOptions, store *loader.Store, engine *core.Engine) (*atomic.Bool, error) { + if r.options.AutomaticScan { + return r.executeSmartWorkflowInput(executerOpts, store, engine) } - if options.MarkdownExportDirectory != "" { - reportingOptions.MarkdownExporter = &markdown.Options{ - Directory: options.MarkdownExportDirectory, - IncludeRawPayload: !options.OmitRawRequests, - SortMode: options.MarkdownExportSortMode, - } - } - if options.SarifExport != "" { - reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport} - } - if options.JSONExport != "" { - reportingOptions.JSONExporter = &jsonexporter.Options{ - File: options.JSONExport, - IncludeRawPayload: !options.OmitRawRequests, - } - } - if options.JSONLExport != "" { - reportingOptions.JSONLExporter = &jsonl.Options{ - File: options.JSONLExport, - IncludeRawPayload: !options.OmitRawRequests, - } - } - - return reportingOptions, nil + return r.executeTemplatesInput(store, engine) } // Close releases all the resources and cleans up @@ -394,6 +341,31 @@ func (r *Runner) Close() { } } +// setupPDCPUpload sets up the PDCP upload writer +// by creating a new writer and returning it +func (r *Runner) setupPDCPUpload(writer output.Writer) output.Writer { + if r.options.DisableCloudUpload || DisableCloudUpload { + r.pdcpUploadErrMsg = fmt.Sprintf("[%v] Scan results upload to cloud is disabled.", aurora.BrightYellow("WRN")) + return writer + } + color := aurora.NewAurora(!r.options.NoColor) + h := &pdcp.PDCPCredHandler{} + creds, err := h.GetCreds() + if err != nil { + if err != pdcp.ErrNoCreds && !HideAutoSaveMsg { + gologger.Verbose().Msgf("Could not get credentials for cloud upload: %s\n", err) + } + r.pdcpUploadErrMsg = fmt.Sprintf("[%v] To view results on Cloud Dashboard, Configure API key from %v", color.BrightYellow("WRN"), pdcp.DashBoardURL) + return writer + } + uploadWriter, err := pdcp.NewUploadWriter(creds) + if err != nil { + r.pdcpUploadErrMsg = fmt.Sprintf("[%v] PDCP (%v) Auto-Save Failed: %s\n", color.BrightYellow("WRN"), pdcp.DashBoardURL, err) + return writer + } + return output.NewMultiWriter(writer, uploadWriter) +} + // RunEnumeration sets up the input layer for giving input nuclei. // binary and runs the actual enumeration func (r *Runner) RunEnumeration() error { @@ -454,25 +426,6 @@ func (r *Runner) RunEnumeration() error { return errors.Wrap(err, "could not load templates from config") } - var cloudTemplates []string - if r.options.Cloud { - // hook template loading - store.NotFoundCallback = func(template string) bool { - parsed, parseErr := strconv.ParseInt(template, 10, 64) - if parseErr != nil { - if err := r.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Type: "templates", Contents: template}); err == nil { - cloudTemplates = append(cloudTemplates, template) - return true - } - return false - } - if exists, err := r.cloudClient.ExistsTemplate(parsed); err == nil { - cloudTemplates = append(cloudTemplates, exists.Reference) - return true - } - return false - } - } if r.options.Validate { if err := store.ValidateTemplates(); err != nil { return err @@ -525,55 +478,8 @@ func (r *Runner) RunEnumeration() error { enumeration := false var results *atomic.Bool - if r.options.Cloud { - if r.options.ScanList { - err = r.getScanList(r.options.OutputLimit) - } else if r.options.DeleteScan != "" { - err = r.deleteScan(r.options.DeleteScan) - } else if r.options.ScanOutput != "" { - err = r.getResults(r.options.ScanOutput, r.options.OutputLimit) - } else if r.options.ListDatasources { - err = r.listDatasources() - } else if r.options.ListTargets { - err = r.listTargets() - } else if r.options.ListTemplates { - err = r.listTemplates() - } else if r.options.ListReportingSources { - err = r.listReportingSources() - } else if r.options.AddDatasource != "" { - err = r.addCloudDataSource(r.options.AddDatasource) - } else if r.options.RemoveDatasource != "" { - err = r.removeDatasource(r.options.RemoveDatasource) - } else if r.options.DisableReportingSource != "" { - err = r.toggleReportingSource(r.options.DisableReportingSource, false) - } else if r.options.EnableReportingSource != "" { - err = r.toggleReportingSource(r.options.EnableReportingSource, true) - } else if r.options.AddTarget != "" { - err = r.addTarget(r.options.AddTarget) - } else if r.options.AddTemplate != "" { - err = r.addTemplate(r.options.AddTemplate) - } else if r.options.GetTarget != "" { - err = r.getTarget(r.options.GetTarget) - } else if r.options.GetTemplate != "" { - err = r.getTemplate(r.options.GetTemplate) - } else if r.options.RemoveTarget != "" { - err = r.removeTarget(r.options.RemoveTarget) - } else if r.options.RemoveTemplate != "" { - err = r.removeTemplate(r.options.RemoveTemplate) - } else if r.options.ReportingConfig != "" { - err = r.addCloudReportingSource() - } else { - if len(store.Templates())+len(store.Workflows())+len(cloudTemplates) == 0 { - return errors.New("no templates provided for scan") - } - gologger.Info().Msgf("Running scan on cloud with URL %s", r.options.CloudURL) - results, err = r.runCloudEnumeration(store, cloudTemplates, r.cloudTargets, r.options.NoStore, r.options.OutputLimit) - enumeration = true - } - } else { - results, err = r.runStandardEnumeration(executorOpts, store, executorEngine) - enumeration = true - } + results, err = r.runStandardEnumeration(executorOpts, store, executorEngine) + enumeration = true if !enumeration { return err @@ -680,6 +586,13 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { gologger.Info().Msgf("Current nuclei version: %v %v", config.Version, updateutils.GetVersionDescription(config.Version, cfg.LatestNucleiVersion)) gologger.Info().Msgf("Current nuclei-templates version: %v %v", cfg.TemplateVersion, updateutils.GetVersionDescription(cfg.TemplateVersion, cfg.LatestNucleiTemplatesVersion)) + if !HideAutoSaveMsg { + if r.pdcpUploadErrMsg != "" { + gologger.Print().Msgf("%s", r.pdcpUploadErrMsg) + } else { + gologger.Info().Msgf("To view results on cloud dashboard, visit %v/scans upon scan completion.", pdcp.DashBoardURL) + } + } if len(store.Templates()) > 0 { gologger.Info().Msgf("New templates added in latest release: %d", len(config.DefaultConfig.GetNewAdditions())) @@ -759,3 +672,8 @@ func expandEndVars(f reflect.Value, fieldType reflect.StructField) { } } } + +func init() { + HideAutoSaveMsg = env.GetEnvOrDefault("DISABLE_CLOUD_UPLOAD_WRN", false) + DisableCloudUpload = env.GetEnvOrDefault("DISABLE_CLOUD_UPLOAD", false) +} diff --git a/pkg/external/customtemplates/templates_provider.go b/pkg/external/customtemplates/templates_provider.go index 213e9b63..471d1648 100644 --- a/pkg/external/customtemplates/templates_provider.go +++ b/pkg/external/customtemplates/templates_provider.go @@ -35,11 +35,6 @@ func (c *CustomTemplatesManager) Update(ctx context.Context) { func NewCustomTemplatesManager(options *types.Options) (*CustomTemplatesManager, error) { ctm := &CustomTemplatesManager{providers: []Provider{}} - if options.Cloud { - // if cloud is enabled, custom templates are Nop - return ctm, nil - } - // Add GitHub providers githubProviders, err := NewGitHubProviders(options) if err != nil { diff --git a/pkg/output/multi_writer.go b/pkg/output/multi_writer.go new file mode 100644 index 00000000..997b00bc --- /dev/null +++ b/pkg/output/multi_writer.go @@ -0,0 +1,59 @@ +package output + +import ( + "github.com/logrusorgru/aurora" +) + +type MultiWriter struct { + writers []Writer +} + +// NewMultiWriter creates a new MultiWriter instance +func NewMultiWriter(writers ...Writer) *MultiWriter { + return &MultiWriter{writers: writers} +} + +func (mw *MultiWriter) Close() { + for _, writer := range mw.writers { + writer.Close() + } +} + +func (mw *MultiWriter) Colorizer() aurora.Aurora { + // Return the colorizer of the first writer + if len(mw.writers) > 0 { + return mw.writers[0].Colorizer() + } + // Default to a no-op colorizer + return aurora.NewAurora(false) +} + +func (mw *MultiWriter) Write(event *ResultEvent) error { + for _, writer := range mw.writers { + if err := writer.Write(event); err != nil { + return err + } + } + return nil +} + +func (mw *MultiWriter) WriteFailure(event *InternalWrappedEvent) error { + for _, writer := range mw.writers { + if err := writer.WriteFailure(event); err != nil { + return err + } + } + return nil +} + +func (mw *MultiWriter) Request(templateID, url, requestType string, err error) { + for _, writer := range mw.writers { + writer.Request(templateID, url, requestType, err) + } +} + +func (mw *MultiWriter) WriteStoreDebugData(host, templateID, eventType string, data string) { + for _, writer := range mw.writers { + writer.WriteStoreDebugData(host, templateID, eventType, data) + } +} diff --git a/pkg/output/output.go b/pkg/output/output.go index f12594a4..56352b20 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -49,20 +49,22 @@ type Writer interface { // StandardWriter is a writer writing output to file and screen for results. type StandardWriter struct { - json bool - jsonReqResp bool - timestamp bool - noMetadata bool - matcherStatus bool - mutex *sync.Mutex - aurora aurora.Aurora - outputFile io.WriteCloser - traceFile io.WriteCloser - errorFile io.WriteCloser - severityColors func(severity.Severity) string - storeResponse bool - storeResponseDir string + json bool + jsonReqResp bool + timestamp bool + noMetadata bool + matcherStatus bool + mutex *sync.Mutex + aurora aurora.Aurora + outputFile io.WriteCloser + traceFile io.WriteCloser + errorFile io.WriteCloser + severityColors func(severity.Severity) string + storeResponse bool + storeResponseDir string omitTemplate bool + DisableStdout bool + AddNewLinesOutputFile bool // by default this is only done for stdout } var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) @@ -243,8 +245,10 @@ func (w *StandardWriter) Write(event *ResultEvent) error { w.mutex.Lock() defer w.mutex.Unlock() - _, _ = os.Stdout.Write(data) - _, _ = os.Stdout.Write([]byte("\n")) + if !w.DisableStdout { + _, _ = os.Stdout.Write(data) + _, _ = os.Stdout.Write([]byte("\n")) + } if w.outputFile != nil { if !w.json { @@ -253,6 +257,9 @@ func (w *StandardWriter) Write(event *ResultEvent) error { if _, writeErr := w.outputFile.Write(data); writeErr != nil { return errors.Wrap(err, "could not write to output") } + if w.AddNewLinesOutputFile && w.json { + _, _ = w.outputFile.Write([]byte("\n")) + } } return nil } diff --git a/pkg/output/standard_writer.go b/pkg/output/standard_writer.go new file mode 100644 index 00000000..41800d7b --- /dev/null +++ b/pkg/output/standard_writer.go @@ -0,0 +1,124 @@ +package output + +import ( + "io" + "os" + "sync" + + "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" + fileutil "github.com/projectdiscovery/utils/file" +) + +// WriterOptions contains configuration options for a writer +type WriterOptions func(s *StandardWriter) error + +// WithJson writes output in json format +func WithJson(json bool, dumpReqResp bool) WriterOptions { + return func(s *StandardWriter) error { + s.json = json + s.jsonReqResp = dumpReqResp + return nil + } +} + +// WithTimestamp writes output with timestamp +func WithTimestamp(timestamp bool) WriterOptions { + return func(s *StandardWriter) error { + s.timestamp = timestamp + return nil + } +} + +// WithNoMetadata disables metadata output +func WithNoMetadata(noMetadata bool) WriterOptions { + return func(s *StandardWriter) error { + s.noMetadata = noMetadata + return nil + } +} + +// WithMatcherStatus writes output with matcher status +func WithMatcherStatus(matcherStatus bool) WriterOptions { + return func(s *StandardWriter) error { + s.matcherStatus = matcherStatus + return nil + } +} + +// WithAurora sets the aurora instance for the writer +func WithAurora(aurora aurora.Aurora) WriterOptions { + return func(s *StandardWriter) error { + s.aurora = aurora + return nil + } +} + +// WithWriter sets the writer for the writer +func WithWriter(outputFile io.WriteCloser) WriterOptions { + return func(s *StandardWriter) error { + s.outputFile = outputFile + return nil + } +} + +// WithTraceSink sets the writer where trace output is written +func WithTraceSink(traceFile io.WriteCloser) WriterOptions { + return func(s *StandardWriter) error { + s.traceFile = traceFile + return nil + } +} + +// WithErrorSink sets the writer where error output is written +func WithErrorSink(errorFile io.WriteCloser) WriterOptions { + return func(s *StandardWriter) error { + s.errorFile = errorFile + return nil + } +} + +// WithSeverityColors sets the color function for severity +func WithSeverityColors(severityColors func(severity.Severity) string) WriterOptions { + return func(s *StandardWriter) error { + s.severityColors = severityColors + return nil + } +} + +// WithStoreResponse sets the store response option +func WithStoreResponse(storeResponse bool, respDir string) WriterOptions { + return func(s *StandardWriter) error { + s.storeResponse = storeResponse + s.storeResponseDir = respDir + return nil + } +} + +// NewWriter creates a new output writer +// if no writer is specified it writes to stdout +func NewWriter(opts ...WriterOptions) (*StandardWriter, error) { + s := &StandardWriter{ + mutex: &sync.Mutex{}, + DisableStdout: true, + AddNewLinesOutputFile: true, + } + for _, opt := range opts { + if err := opt(s); err != nil { + return nil, err + } + } + if s.aurora == nil { + s.aurora = aurora.NewAurora(false) + } + if s.outputFile == nil { + s.outputFile = os.Stdout + } + // Try to create output folder if it doesn't exist + if s.storeResponse && !fileutil.FolderExists(s.storeResponseDir) { + if err := fileutil.CreateFolder(s.storeResponseDir); err != nil { + return nil, err + } + } + return s, nil +} diff --git a/pkg/testutils/integration.go b/pkg/testutils/integration.go index 91a5adc1..8b5a71b1 100644 --- a/pkg/testutils/integration.go +++ b/pkg/testutils/integration.go @@ -17,7 +17,13 @@ import ( ) // ExtraArgs -var ExtraDebugArgs = []string{} +var ( + ExtraDebugArgs = []string{} + ExtraEnvVars = []string{ + "DISABLE_CLOUD_UPLOAD_WRN=true", + "DISABLE_CLOUD_UPLOAD=true", + } +) // RunNucleiTemplateAndGetResults returns a list of results for a template func RunNucleiTemplateAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { @@ -56,6 +62,7 @@ func RunNucleiBareArgsAndGetResults(debug bool, env []string, extra ...string) ( if env != nil { cmd.Env = append(os.Environ(), env...) } + cmd.Env = append(cmd.Env, ExtraEnvVars...) if debug { cmd.Args = append(cmd.Args, "-debug") cmd.Stderr = os.Stderr @@ -83,6 +90,7 @@ func RunNucleiBareArgsAndGetResults(debug bool, env []string, extra ...string) ( // RunNucleiArgsAndGetResults returns result,and runtime errors func RunNucleiWithArgsAndGetResults(debug bool, args ...string) ([]string, error) { cmd := exec.Command("./nuclei", args...) + cmd.Env = append(cmd.Env, ExtraEnvVars...) if debug { cmd.Args = append(cmd.Args, "-debug") cmd.Stderr = os.Stderr @@ -118,6 +126,7 @@ func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]str cmd.Args = append(cmd.Args, "-interactions-cooldown-period", "10") cmd.Args = append(cmd.Args, "-allow-local-file-access") cmd.Args = append(cmd.Args, "-nc") // disable color + cmd.Env = append(cmd.Env, ExtraEnvVars...) data, err := cmd.CombinedOutput() if debug { fmt.Println(string(data)) @@ -142,6 +151,7 @@ func RunNucleiArgsWithEnvAndGetResults(debug bool, env []string, extra ...string cmd := exec.Command("./nuclei") extra = append(extra, ExtraDebugArgs...) cmd.Env = append(os.Environ(), env...) + cmd.Env = append(cmd.Env, ExtraEnvVars...) cmd.Args = append(cmd.Args, extra...) cmd.Args = append(cmd.Args, "-duc") // disable auto updates cmd.Args = append(cmd.Args, "-interactions-poll-duration", "5") @@ -176,6 +186,7 @@ var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded[^:]*: (\ // RunNucleiBinaryAndGetLoadedTemplates returns a list of results for a template func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, debug bool, args []string) (string, error) { cmd := exec.Command(nucleiBinary, args...) + cmd.Env = append(cmd.Env, ExtraEnvVars...) cmd.Args = append(cmd.Args, "-duc") // disable auto updates if debug { cmd.Args = append(cmd.Args, "-debug") @@ -197,6 +208,7 @@ func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, debug bool, args func RunNucleiBinaryAndGetCombinedOutput(debug bool, args []string) (string, error) { args = append(args, "-interactions-cooldown-period", "10", "-interactions-poll-duration", "1") cmd := exec.Command("./nuclei", args...) + cmd.Env = append(cmd.Env, ExtraEnvVars...) if debug { cmd.Args = append(cmd.Args, "-debug") fmt.Println(cmd.String()) diff --git a/pkg/types/types.go b/pkg/types/types.go index f73edb5a..fa6c11b6 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -104,48 +104,6 @@ type Options struct { MarkdownExportSortMode string // SarifExport is the file to export sarif output format to SarifExport string - // CloudURL is the URL for the nuclei cloud endpoint - CloudURL string - // CloudAPIKey is the api-key for the nuclei cloud endpoint - CloudAPIKey string - // ScanList feature to get all the scan ids for a user - ScanList bool - // ListDatasources enables listing of datasources for user - ListDatasources bool - // ListTargets enables listing of targets for user - ListTargets bool - // ListTemplates enables listing of templates for user - ListTemplates bool - // ListReportingSources enables listing of reporting source - ListReportingSources bool - // DisableReportingSource disables a reporting source - DisableReportingSource string - // EnableReportingSource enables a reporting source - EnableReportingSource string - // Limit the number of items at a time - OutputLimit int - // Nostore - NoStore bool - // Delete scan - DeleteScan string - // AddDatasource adds a datasource to cloud storage - AddDatasource string - // RemoveDatasource deletes a datasource from cloud storage - RemoveDatasource string - // AddTemplate adds a list of templates to custom datasource - AddTemplate string - // AddTarget adds a list of targets to custom datasource - AddTarget string - // GetTemplate gets a template by id - GetTemplate string - // GetTarget gets a target by id - GetTarget string - // RemoveTemplate removes a list of templates - RemoveTemplate string - // RemoveTarget removes a list of targets - RemoveTarget string - // Get issues for a scan - ScanOutput string // ResolversFile is a file containing resolvers for nuclei. ResolversFile string // StatsInterval is the number of seconds to display stats after @@ -206,8 +164,6 @@ type Options struct { ShowBrowser bool // HeadlessOptionalArguments specifies optional arguments to pass to Chrome HeadlessOptionalArguments goflags.StringSlice - // NoTables disables pretty printing of cloud results in tables - NoTables bool // DisableClustering disables clustering of templates DisableClustering bool // UseInstalledChrome skips chrome install and use local instance @@ -258,8 +214,6 @@ type Options struct { JSONExport string // JSONLExport is the file to export JSONL output format to JSONLExport string - // Cloud enables nuclei cloud scan execution - Cloud bool // EnableProgressBar enables progress bar EnableProgressBar bool // TemplateDisplay displays the template contents @@ -402,6 +356,8 @@ type Options struct { SignTemplates bool // EnableCodeTemplates enables code templates EnableCodeTemplates bool + // Disables cloud upload + DisableCloudUpload bool } // ShouldLoadResume resume file @@ -440,23 +396,6 @@ func DefaultOptions() *Options { } } -// HasCloudOptions returns true if cloud options have been specified -func (options *Options) HasCloudOptions() bool { - return options.ScanList || - options.DeleteScan != "" || - options.ScanOutput != "" || - options.ListDatasources || - options.ListTargets || - options.ListTemplates || - options.RemoveDatasource != "" || - options.AddTarget != "" || - options.AddTemplate != "" || - options.RemoveTarget != "" || - options.RemoveTemplate != "" || - options.GetTarget != "" || - options.GetTemplate != "" -} - func (options *Options) ShouldUseHostError() bool { return options.MaxHostError > 0 && !options.NoHostErrors }