Merge pull request #109 from projectdiscovery/iceman-update

Added update templates feature to nuclei to download and manage templates
dev
Ice3man 2020-06-25 08:58:38 -07:00 committed by GitHub
commit 3a24879aeb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 328 additions and 17 deletions

3
go.mod
View File

@ -5,6 +5,9 @@ go 1.14
require (
github.com/Knetic/govaluate v3.0.0+incompatible
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
github.com/blang/semver v3.5.1+incompatible
github.com/google/go-github/v32 v32.0.0
github.com/json-iterator/go v1.1.10
github.com/karrick/godirwalk v1.15.6
github.com/miekg/dns v1.1.29
github.com/pkg/errors v0.9.1

21
go.sum
View File

@ -3,13 +3,30 @@ github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f
github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
github.com/google/go-github/v32 v32.0.0 h1:q74KVb22spUq0U5HqZ9VCYqQz8YRuOtL/39ZnfwO+NM=
github.com/google/go-github/v32 v32.0.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA=
github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@ -20,16 +37,19 @@ github.com/projectdiscovery/retryabledns v1.0.4/go.mod h1:/UzJn4I+cPdQl6pKiiQfvV
github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200528225125-3c3fba18258b h1:IYiJPiJfzktmDAO1HQiwjMjwjlYKHAL7KzeD544RJPs=
golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -39,6 +59,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=

View File

@ -11,20 +11,22 @@ import (
// Options contains the configuration options for tuning
// the template requesting process.
type Options struct {
Debug bool // Debug mode allows debugging request/responses for the engine
Templates string // Signature specifies the template/templates to use
Targets string // Targets specifies the targets to scan using templates.
Threads int // Thread controls the number of concurrent requests to make.
Timeout int // Timeout is the seconds to wait for a response from the server.
Retries int // Retries is the number of times to retry the request
Output string // Output is the file to write found subdomains to.
ProxyURL string // ProxyURL is the URL for the proxy server
ProxySocksURL string // ProxySocksURL is the URL for the proxy socks server
Silent bool // Silent suppresses any extra text and only writes found URLs on screen.
Version bool // Version specifies if we should just show version and exit
Verbose bool // Verbose flag indicates whether to show verbose output or not
NoColor bool // No-Color disables the colored output.
CustomHeaders requests.CustomHeaders // Custom global headers
Debug bool // Debug mode allows debugging request/responses for the engine
Templates string // Signature specifies the template/templates to use
Targets string // Targets specifies the targets to scan using templates.
Threads int // Thread controls the number of concurrent requests to make.
Timeout int // Timeout is the seconds to wait for a response from the server.
Retries int // Retries is the number of times to retry the request
Output string // Output is the file to write found subdomains to.
ProxyURL string // ProxyURL is the URL for the proxy server
ProxySocksURL string // ProxySocksURL is the URL for the proxy socks server
Silent bool // Silent suppresses any extra text and only writes found URLs on screen.
Version bool // Version specifies if we should just show version and exit
Verbose bool // Verbose flag indicates whether to show verbose output or not
NoColor bool // No-Color disables the colored output.
CustomHeaders requests.CustomHeaders // Custom global headers
UpdateTemplates bool // UpdateTemplates updates the templates installed at startup
TemplatesDirectory string // TemplatesDirectory is the directory to use for storing templates
Stdin bool // Stdin specifies whether stdin input was given to the process
}
@ -47,6 +49,8 @@ func ParseOptions() *Options {
flag.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
flag.Var(&options.CustomHeaders, "H", "Custom Header.")
flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses")
flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)")
flag.StringVar(&options.TemplatesDirectory, "templates-directory", "", "Directory to use for storing nuclei-templates")
flag.Parse()

View File

@ -23,7 +23,8 @@ type Runner struct {
output *os.File
outputMutex *sync.Mutex
tempFile string
tempFile string
templatesConfig *nucleiConfig
// options contains configuration options for runner
options *Options
}
@ -35,6 +36,13 @@ func New(options *Options) (*Runner, error) {
options: options,
}
if err := runner.updateTemplates(); err != nil {
return nil, err
}
if (options.Templates == "" || options.Targets == "" && !options.Stdin) && options.UpdateTemplates {
os.Exit(0)
}
// If we have stdin, write it to a new file
if options.Stdin {
tempInput, err := ioutil.TempFile("", "stdin-input-*")

275
internal/runner/update.go Normal file
View File

@ -0,0 +1,275 @@
package runner
import (
"archive/zip"
"bytes"
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
"regexp"
"strings"
"github.com/blang/semver"
"github.com/google/go-github/v32/github"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/gologger"
)
// nucleiConfig contains some configuration options for nuclei
type nucleiConfig struct {
TemplatesDirectory string `json:"templates-directory,omitempty"`
CurrentVersion string `json:"current-version,omitempty"`
}
// nucleiConfigFilename is the filename of nuclei configuration file.
const nucleiConfigFilename = ".nuclei-config.json"
var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
// readConfiguration reads the nuclei configuration file from disk.
func (r *Runner) readConfiguration() (*nucleiConfig, error) {
home, err := os.UserHomeDir()
if err != nil {
return nil, err
}
templatesConfigFile := path.Join(home, nucleiConfigFilename)
file, err := os.Open(templatesConfigFile)
if err != nil {
return nil, err
}
defer file.Close()
config := &nucleiConfig{}
if err = jsoniter.NewDecoder(file).Decode(config); err != nil {
return nil, err
}
return config, nil
}
// readConfiguration reads the nuclei configuration file from disk.
func (r *Runner) writeConfiguration(config *nucleiConfig) error {
home, err := os.UserHomeDir()
if err != nil {
return err
}
templatesConfigFile := path.Join(home, nucleiConfigFilename)
file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
if err != nil {
return err
}
defer file.Close()
if err = jsoniter.NewEncoder(file).Encode(config); err != nil {
return err
}
return nil
}
// updateTemplates checks if the default list of nuclei-templates
// exist in the users home directory, if not the latest revision
// is downloaded from github.
//
// If the path exists but is not latest, the new version is downloaded
// from github and replaced with the templates directory.
func (r *Runner) updateTemplates() error {
home, err := os.UserHomeDir()
if err != nil {
return err
}
templatesConfigFile := path.Join(home, nucleiConfigFilename)
if _, err := os.Stat(templatesConfigFile); !os.IsNotExist(err) {
config, err := r.readConfiguration()
if err != nil {
return err
}
r.templatesConfig = config
}
if r.templatesConfig == nil || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
if !r.options.UpdateTemplates {
gologger.Warningf("nuclei-templates are not installed, use update-templates flag.\n")
return nil
}
// Use custom location if user has given a template directory
if r.options.TemplatesDirectory != "" {
home = r.options.TemplatesDirectory
}
r.templatesConfig = &nucleiConfig{TemplatesDirectory: path.Join(home, "nuclei-templates")}
os.RemoveAll(r.templatesConfig.TemplatesDirectory)
// Download the repository and also write the revision to a HEAD file.
version, asset, err := r.getLatestReleaseFromGithub()
if err != nil {
return err
}
gologger.Verbosef("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory)
if err = r.downloadReleaseAndUnzip(asset.GetZipballURL()); err != nil {
return err
}
r.templatesConfig.CurrentVersion = version.String()
if err = r.writeConfiguration(r.templatesConfig); err != nil {
return err
}
gologger.Infof("Successfully downloaded nuclei-templates (v%s). Enjoy!\n", version.String())
return nil
}
// Get the configuration currently on disk.
verText := r.templatesConfig.CurrentVersion
indices := reVersion.FindStringIndex(verText)
if indices == nil {
return fmt.Errorf("invalid release found with tag %s", err)
}
if indices[0] > 0 {
verText = verText[indices[0]:]
}
oldVersion, err := semver.Make(verText)
if err != nil {
return err
}
version, asset, err := r.getLatestReleaseFromGithub()
if err != nil {
return err
}
if version.EQ(oldVersion) {
gologger.Verbosef("Latest version of nuclei-templates installed: v%s\n", "update-templates", oldVersion.String())
return nil
}
if version.GT(oldVersion) {
if !r.options.UpdateTemplates {
gologger.Warningf("You're using outdated nuclei-templates. Latest v%s\n", version.String())
return nil
}
if r.options.TemplatesDirectory != "" {
home = r.options.TemplatesDirectory
r.templatesConfig.TemplatesDirectory = path.Join(home, "nuclei-templates")
}
r.templatesConfig.CurrentVersion = version.String()
os.RemoveAll(r.templatesConfig.TemplatesDirectory)
gologger.Verbosef("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory)
if err = r.downloadReleaseAndUnzip(asset.GetZipballURL()); err != nil {
return err
}
if err = r.writeConfiguration(r.templatesConfig); err != nil {
return err
}
gologger.Infof("Successfully updated nuclei-templates (v%s). Enjoy!\n", version.String())
}
return nil
}
const (
userName = "projectdiscovery"
repoName = "nuclei-templates"
)
// getLatestReleaseFromGithub returns the latest release from github
func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.RepositoryRelease, error) {
client := github.NewClient(nil)
rels, _, err := client.Repositories.ListReleases(context.Background(), userName, repoName, nil)
if err != nil {
return semver.Version{}, nil, err
}
// Find the most recent version based on semantic versioning.
var latestRelease semver.Version
var latestPublish *github.RepositoryRelease
for _, release := range rels {
verText := release.GetTagName()
indices := reVersion.FindStringIndex(verText)
if indices == nil {
return semver.Version{}, nil, fmt.Errorf("invalid release found with tag %s", err)
}
if indices[0] > 0 {
verText = verText[indices[0]:]
}
ver, err := semver.Make(verText)
if err != nil {
return semver.Version{}, nil, err
}
if latestPublish == nil || ver.GTE(latestRelease) {
latestRelease = ver
latestPublish = release
}
}
if latestPublish == nil {
return semver.Version{}, nil, errors.New("no version found for the templates")
}
return latestRelease, latestPublish, nil
}
// downloadReleaseAndUnzip downloads and unzips the release in a directory
func (r *Runner) downloadReleaseAndUnzip(downloadURL string) error {
req, err := http.NewRequest("GET", downloadURL, nil)
if err != nil {
return fmt.Errorf("Failed to create HTTP request to %s: %s", downloadURL, err)
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("Failed to download a release file from %s: %s", downloadURL, err)
}
defer res.Body.Close()
if res.StatusCode != 200 {
return fmt.Errorf("Failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode)
}
buf, err := ioutil.ReadAll(res.Body)
if err != nil {
return fmt.Errorf("Failed to create buffer for zip file: %s", err)
}
reader := bytes.NewReader(buf)
z, err := zip.NewReader(reader, reader.Size())
if err != nil {
return fmt.Errorf("Failed to uncompress zip file: %s", err)
}
// Create the template folder if it doesn't exists
os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm)
for _, file := range z.File {
directory, name := filepath.Split(file.Name)
if name == "" {
continue
}
paths := strings.Split(directory, "/")
finalPath := strings.Join(paths[1:], "/")
templateDirectory := path.Join(r.templatesConfig.TemplatesDirectory, finalPath)
os.MkdirAll(templateDirectory, os.ModePerm)
f, err := os.Create(path.Join(templateDirectory, name))
if err != nil {
return fmt.Errorf("Could not create uncompressed file: %s", err)
}
defer f.Close()
reader, err := file.Open()
if err != nil {
return fmt.Errorf("Could not open archive to extract file: %s", err)
}
io.Copy(f, reader)
}
return nil
}

View File

@ -15,11 +15,11 @@ func (options *Options) validateOptions() error {
}
// Check if a list of resolvers was provided and it exists
if options.Templates == "" {
if options.Templates == "" && !options.UpdateTemplates {
return errors.New("no template/templates provided")
}
if options.Targets == "" && !options.Stdin {
if options.Targets == "" && !options.Stdin && !options.UpdateTemplates {
return errors.New("no target input provided")
}