From cb39fd9d13e1fcde70795ba26f3f762100a2820e Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 11 Sep 2021 18:40:07 +0530 Subject: [PATCH] Use separate nuclei version check infra --- v2/go.mod | 1 + v2/go.sum | 4 + v2/internal/runner/update.go | 130 ++++++-------------------------- v2/pkg/catalog/config/config.go | 4 - 4 files changed, 30 insertions(+), 109 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 1c90069e..500b34b5 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -107,6 +107,7 @@ require ( github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46 // indirect github.com/projectdiscovery/mapcidr v0.0.6 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect + github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210911130026-62ec404ee755 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/tklauser/go-sysconf v0.3.7 // indirect github.com/tklauser/numcpus v0.2.3 // indirect diff --git a/v2/go.sum b/v2/go.sum index ba69b504..be72b967 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -224,6 +224,7 @@ github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxC github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs= github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -350,6 +351,9 @@ github.com/projectdiscovery/mapcidr v0.0.6 h1:RRIrqNakUEF/pstIXWTD6yvCMF9N6SnOb9 github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= +github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210911130026-62ec404ee755 h1:qFZ9wpHf90mx+KmqIlvoDOxLlwOuJe0DyOICk7dl91A= +github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210911130026-62ec404ee755/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= +github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY= github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg= github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index a0aa2bd3..83b19354 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -7,7 +7,6 @@ import ( "context" "crypto/md5" "encoding/hex" - "encoding/json" "fmt" "io" "io/ioutil" @@ -28,6 +27,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei-updatecheck-api/client" "github.com/tj/go-update" "github.com/tj/go-update/progress" githubUpdateStore "github.com/tj/go-update/stores/github" @@ -38,7 +38,6 @@ const ( repoName = "nuclei-templates" nucleiIgnoreFile = ".nuclei-ignore" nucleiConfigFilename = ".templates-config.json" - defaultIgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore" ) var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`) @@ -65,7 +64,6 @@ func (r *Runner) updateTemplates() error { if r.templatesConfig == nil { currentConfig := &config.Config{ TemplatesDirectory: filepath.Join(home, "nuclei-templates"), - IgnoreURL: defaultIgnoreURL, NucleiVersion: config.Version, } if writeErr := config.WriteConfiguration(currentConfig, false, false); writeErr != nil { @@ -99,15 +97,15 @@ func (r *Runner) updateTemplates() error { if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != filepath.Join(home, "nuclei-templates") { r.templatesConfig.TemplatesDirectory, _ = filepath.Abs(r.options.TemplatesDirectory) } + r.fetchLatestVersionsFromGithub() // also fetch latest versions // Download the repository and also write the revision to a HEAD file. - version, asset, getErr := r.getLatestReleaseFromGithub() + version, asset, getErr := r.getLatestReleaseFromGithub(r.templatesConfig.NucleiTemplatesLatestVersion) if getErr != nil { return getErr } gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) - r.fetchLatestVersionsFromGithub() // also fetch latest versions _, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()) if err != nil { return err @@ -127,6 +125,7 @@ func (r *Runner) updateTemplates() error { if time.Since(r.templatesConfig.LastChecked) < 24*time.Hour && !r.options.UpdateTemplates { return nil } + r.fetchLatestVersionsFromGithub() // also fetch latest versions // Get the configuration currently on disk. verText := r.templatesConfig.CurrentVersion @@ -143,7 +142,7 @@ func (r *Runner) updateTemplates() error { return err } - version, asset, err := r.getLatestReleaseFromGithub() + version, asset, err := r.getLatestReleaseFromGithub(r.templatesConfig.NucleiLatestVersion) if err != nil { return err } @@ -195,75 +194,37 @@ func (r *Runner) readInternalConfigurationFile(home, configDir string) error { // checkNucleiIgnoreFileUpdates checks .nuclei-ignore file for updates from github func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool { - ignoreURL := defaultIgnoreURL - if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" { - ignoreURL = r.templatesConfig.IgnoreURL + data, err := client.GetLatestIgnoreFile() + if err != nil { + return false } - gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL) - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil) - if reqErr == nil { - resp, httpGetErr := http.DefaultClient.Do(req) - if httpGetErr != nil { - if resp != nil && resp.Body != nil { - resp.Body.Close() - } - gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, httpGetErr) - } else { - data, _ := ioutil.ReadAll(resp.Body) - resp.Body.Close() - - if len(data) > 0 { - _ = ioutil.WriteFile(filepath.Join(configDir, nucleiIgnoreFile), data, 0644) - } - if r.templatesConfig != nil { - if err := config.WriteConfiguration(r.templatesConfig, false, true); err != nil { - gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err) - } - } + if len(data) > 0 { + _ = ioutil.WriteFile(filepath.Join(configDir, nucleiIgnoreFile), data, 0644) + } + if r.templatesConfig != nil { + if err := config.WriteConfiguration(r.templatesConfig, false, true); err != nil { + gologger.Warning().Msgf("Could not get ignore-file from server: %s", err) } } - cancel() return true } // getLatestReleaseFromGithub returns the latest release from github -func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.RepositoryRelease, error) { +func (r *Runner) getLatestReleaseFromGithub(latestTag string) (semver.Version, *github.RepositoryRelease, error) { client := github.NewClient(nil) - rels, _, err := client.Repositories.ListReleases(context.Background(), userName, repoName, nil) + parsed, err := semver.Parse(latestTag) if err != nil { return semver.Version{}, nil, err } - - // Find the most recent version based on semantic versioning. - var latestRelease semver.Version - var latestPublish *github.RepositoryRelease - for _, release := range rels { - verText := release.GetTagName() - indices := reVersion.FindStringIndex(verText) - if indices == nil { - return semver.Version{}, nil, fmt.Errorf("invalid release found with tag %s", err) - } - if indices[0] > 0 { - verText = verText[indices[0]:] - } - - ver, err := semver.Make(verText) - if err != nil { - return semver.Version{}, nil, err - } - - if latestPublish == nil || ver.GTE(latestRelease) { - latestRelease = ver - latestPublish = release - } + release, _, err := client.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag) + if err != nil { + return semver.Version{}, nil, err } - if latestPublish == nil { + if release == nil { return semver.Version{}, nil, errors.New("no version found for the templates") } - return latestRelease, latestPublish, nil + return parsed, release, nil } // downloadReleaseAndUnzip downloads and unzips the release in a directory @@ -504,57 +465,16 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st // fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github func (r *Runner) fetchLatestVersionsFromGithub() { - nucleiLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei") + versions, err := client.GetLatestNucleiTemplatesVersion() if err != nil { - gologger.Warning().Msgf("Could not fetch latest nuclei release: %s", err) - } - templatesLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei-templates") - if err != nil { - gologger.Warning().Msgf("Could not fetch latest nuclei-templates release: %s", err) + gologger.Warning().Msgf("Could not fetch latest releases: %s", err) } if r.templatesConfig != nil { - r.templatesConfig.NucleiLatestVersion = nucleiLatest - r.templatesConfig.NucleiTemplatesLatestVersion = templatesLatest + r.templatesConfig.NucleiLatestVersion = versions.Nuclei + r.templatesConfig.NucleiTemplatesLatestVersion = versions.Templates } } -type githubTagData struct { - Name string -} - -// githubFetchLatestTagRepo fetches latest tag from github -// This function was half written by github copilot AI :D. -func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - url := fmt.Sprintf("https://api.github.com/repos/%s/tags", repo) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return "", err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - - var tags []githubTagData - err = json.Unmarshal(body, &tags) - if err != nil { - return "", err - } - if len(tags) == 0 { - return "", fmt.Errorf("no tags found for %s", repo) - } - return strings.TrimPrefix(tags[0].Name, "v"), nil -} - // updateNucleiVersionToLatest implements nuclei auto-updation using Github Releases. func updateNucleiVersionToLatest(verbose bool) error { if verbose { diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index cfb7148f..598ccf35 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -16,7 +16,6 @@ type Config struct { TemplatesDirectory string `json:"templates-directory,omitempty"` CurrentVersion string `json:"current-version,omitempty"` LastChecked time.Time `json:"last-checked,omitempty"` - IgnoreURL string `json:"ignore-url,omitempty"` NucleiVersion string `json:"nuclei-version,omitempty"` LastCheckedIgnore time.Time `json:"last-checked-ignore,omitempty"` @@ -64,9 +63,6 @@ func ReadConfiguration() (*Config, error) { // WriteConfiguration writes the updated nuclei configuration to disk func WriteConfiguration(config *Config, checked, checkedIgnore bool) error { - if config.IgnoreURL == "" { - config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore" - } if checked { config.LastChecked = time.Now() }