Use separate nuclei version check infra

2021-09-11 18:40:07 +05:30 · 2021-09-11 18:40:07 +05:30 · cb39fd9d13
parent dfff092caf
commit cb39fd9d13
4 changed files with 30 additions and 109 deletions
--- a/v2/go.mod
+++ b/v2/go.mod
@ -107,6 +107,7 @@ require (
 	github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46 // indirect
 	github.com/projectdiscovery/mapcidr v0.0.6 // indirect
 	github.com/projectdiscovery/networkpolicy v0.0.1 // indirect
+	github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210911130026-62ec404ee755 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.7 // indirect
 	github.com/tklauser/numcpus v0.2.3 // indirect
--- a/v2/go.sum
+++ b/v2/go.sum
@ -224,6 +224,7 @@ github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxC
 github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
 github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
@ -350,6 +351,9 @@ github.com/projectdiscovery/mapcidr v0.0.6 h1:RRIrqNakUEF/pstIXWTD6yvCMF9N6SnOb9
 github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM=
 github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I=
 github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs=
+github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210911130026-62ec404ee755 h1:qFZ9wpHf90mx+KmqIlvoDOxLlwOuJe0DyOICk7dl91A=
+github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210911130026-62ec404ee755/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M=
+github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY=
 github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg=
 github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
 github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
--- a/v2/internal/runner/update.go
+++ b/v2/internal/runner/update.go
@ -7,7 +7,6 @@ import (
 	"context"
 	"crypto/md5"
 	"encoding/hex"
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@ -28,6 +27,7 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"

+	"github.com/projectdiscovery/nuclei-updatecheck-api/client"
 	"github.com/tj/go-update"
 	"github.com/tj/go-update/progress"
 	githubUpdateStore "github.com/tj/go-update/stores/github"
@ -38,7 +38,6 @@ const (
 	repoName             = "nuclei-templates"
 	nucleiIgnoreFile     = ".nuclei-ignore"
 	nucleiConfigFilename = ".templates-config.json"
-	defaultIgnoreURL     = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
 )

 var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
@ -65,7 +64,6 @@ func (r *Runner) updateTemplates() error {
 	if r.templatesConfig == nil {
 		currentConfig := &config.Config{
 			TemplatesDirectory: filepath.Join(home, "nuclei-templates"),
-			IgnoreURL:          defaultIgnoreURL,
 			NucleiVersion:      config.Version,
 		}
 		if writeErr := config.WriteConfiguration(currentConfig, false, false); writeErr != nil {
@ -99,15 +97,15 @@ func (r *Runner) updateTemplates() error {
 		if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != filepath.Join(home, "nuclei-templates") {
 			r.templatesConfig.TemplatesDirectory, _ = filepath.Abs(r.options.TemplatesDirectory)
 		}
+		r.fetchLatestVersionsFromGithub() // also fetch latest versions

 		// Download the repository and also write the revision to a HEAD file.
-		version, asset, getErr := r.getLatestReleaseFromGithub()
+		version, asset, getErr := r.getLatestReleaseFromGithub(r.templatesConfig.NucleiTemplatesLatestVersion)
 		if getErr != nil {
 			return getErr
 		}
 		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)

-		r.fetchLatestVersionsFromGithub() // also fetch latest versions
 		_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
 		if err != nil {
 			return err
@ -127,6 +125,7 @@ func (r *Runner) updateTemplates() error {
 	if time.Since(r.templatesConfig.LastChecked) < 24*time.Hour && !r.options.UpdateTemplates {
 		return nil
 	}
+	r.fetchLatestVersionsFromGithub() // also fetch latest versions

 	// Get the configuration currently on disk.
 	verText := r.templatesConfig.CurrentVersion
@ -143,7 +142,7 @@ func (r *Runner) updateTemplates() error {
 		return err
 	}

-	version, asset, err := r.getLatestReleaseFromGithub()
+	version, asset, err := r.getLatestReleaseFromGithub(r.templatesConfig.NucleiLatestVersion)
 	if err != nil {
 		return err
 	}
@ -195,75 +194,37 @@ func (r *Runner) readInternalConfigurationFile(home, configDir string) error {

 // checkNucleiIgnoreFileUpdates checks .nuclei-ignore file for updates from github
 func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool {
-	ignoreURL := defaultIgnoreURL
-	if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" {
-		ignoreURL = r.templatesConfig.IgnoreURL
+	data, err := client.GetLatestIgnoreFile()
+	if err != nil {
+		return false
 	}
-	gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
-	if reqErr == nil {
-		resp, httpGetErr := http.DefaultClient.Do(req)
-		if httpGetErr != nil {
-			if resp != nil && resp.Body != nil {
-				resp.Body.Close()
-			}
-			gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, httpGetErr)
-		} else {
-			data, _ := ioutil.ReadAll(resp.Body)
-			resp.Body.Close()
-
-			if len(data) > 0 {
-				_ = ioutil.WriteFile(filepath.Join(configDir, nucleiIgnoreFile), data, 0644)
-			}
-			if r.templatesConfig != nil {
-				if err := config.WriteConfiguration(r.templatesConfig, false, true); err != nil {
-					gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
-				}
-			}
+	if len(data) > 0 {
+		_ = ioutil.WriteFile(filepath.Join(configDir, nucleiIgnoreFile), data, 0644)
+	}
+	if r.templatesConfig != nil {
+		if err := config.WriteConfiguration(r.templatesConfig, false, true); err != nil {
+			gologger.Warning().Msgf("Could not get ignore-file from server: %s", err)
 		}
 	}
-	cancel()
 	return true
 }

 // getLatestReleaseFromGithub returns the latest release from github
-func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.RepositoryRelease, error) {
+func (r *Runner) getLatestReleaseFromGithub(latestTag string) (semver.Version, *github.RepositoryRelease, error) {
 	client := github.NewClient(nil)

-	rels, _, err := client.Repositories.ListReleases(context.Background(), userName, repoName, nil)
+	parsed, err := semver.Parse(latestTag)
 	if err != nil {
 		return semver.Version{}, nil, err
 	}
-
-	// Find the most recent version based on semantic versioning.
-	var latestRelease semver.Version
-	var latestPublish *github.RepositoryRelease
-	for _, release := range rels {
-		verText := release.GetTagName()
-		indices := reVersion.FindStringIndex(verText)
-		if indices == nil {
-			return semver.Version{}, nil, fmt.Errorf("invalid release found with tag %s", err)
-		}
-		if indices[0] > 0 {
-			verText = verText[indices[0]:]
-		}
-
-		ver, err := semver.Make(verText)
-		if err != nil {
-			return semver.Version{}, nil, err
-		}
-
-		if latestPublish == nil || ver.GTE(latestRelease) {
-			latestRelease = ver
-			latestPublish = release
-		}
+	release, _, err := client.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag)
+	if err != nil {
+		return semver.Version{}, nil, err
 	}
-	if latestPublish == nil {
+	if release == nil {
 		return semver.Version{}, nil, errors.New("no version found for the templates")
 	}
-	return latestRelease, latestPublish, nil
+	return parsed, release, nil
 }

 // downloadReleaseAndUnzip downloads and unzips the release in a directory
@ -504,57 +465,16 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st

 // fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github
 func (r *Runner) fetchLatestVersionsFromGithub() {
-	nucleiLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei")
+	versions, err := client.GetLatestNucleiTemplatesVersion()
 	if err != nil {
-		gologger.Warning().Msgf("Could not fetch latest nuclei release: %s", err)
-	}
-	templatesLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei-templates")
-	if err != nil {
-		gologger.Warning().Msgf("Could not fetch latest nuclei-templates release: %s", err)
+		gologger.Warning().Msgf("Could not fetch latest releases: %s", err)
 	}
 	if r.templatesConfig != nil {
-		r.templatesConfig.NucleiLatestVersion = nucleiLatest
-		r.templatesConfig.NucleiTemplatesLatestVersion = templatesLatest
+		r.templatesConfig.NucleiLatestVersion = versions.Nuclei
+		r.templatesConfig.NucleiTemplatesLatestVersion = versions.Templates
 	}
 }

-type githubTagData struct {
-	Name string
-}
-
-// githubFetchLatestTagRepo fetches latest tag from github
-// This function was half written by github copilot AI :D.
-func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	url := fmt.Sprintf("https://api.github.com/repos/%s/tags", repo)
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
-	if err != nil {
-		return "", err
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-
-	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return "", err
-	}
-
-	var tags []githubTagData
-	err = json.Unmarshal(body, &tags)
-	if err != nil {
-		return "", err
-	}
-	if len(tags) == 0 {
-		return "", fmt.Errorf("no tags found for %s", repo)
-	}
-	return strings.TrimPrefix(tags[0].Name, "v"), nil
-}
-
 // updateNucleiVersionToLatest implements nuclei auto-updation using Github Releases.
 func updateNucleiVersionToLatest(verbose bool) error {
 	if verbose {
--- a/v2/pkg/catalog/config/config.go
+++ b/v2/pkg/catalog/config/config.go
@ -16,7 +16,6 @@ type Config struct {
 	TemplatesDirectory string    `json:"templates-directory,omitempty"`
 	CurrentVersion     string    `json:"current-version,omitempty"`
 	LastChecked        time.Time `json:"last-checked,omitempty"`
-	IgnoreURL          string    `json:"ignore-url,omitempty"`
 	NucleiVersion      string    `json:"nuclei-version,omitempty"`
 	LastCheckedIgnore  time.Time `json:"last-checked-ignore,omitempty"`

@ -64,9 +63,6 @@ func ReadConfiguration() (*Config, error) {

 // WriteConfiguration writes the updated nuclei configuration to disk
 func WriteConfiguration(config *Config, checked, checkedIgnore bool) error {
-	if config.IgnoreURL == "" {
-		config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
-	}
 	if checked {
 		config.LastChecked = time.Now()
 	}