use std config directories (#4228)

* use std config directories

* use NUCLEI_CONFIG_DIR env for config dir

* add template sources option in sdk

* add cloud.projectdiscovery.io to trusted domain

* fix failing test
dev
Tarun Koyalwar 2023-10-13 11:55:09 +05:30 committed by GitHub
parent 39d094075b
commit 9a39757caa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 191 additions and 64 deletions

View File

@ -20,4 +20,4 @@ code:
- type: word
words:
- "hello from input baz"
# digest: 4a0a004730450221009b73a0d173beca99aff61fcfe366d929760411f0b9facfe5502a0f2e677342fa02201327b024e52a89891bb90ecdd2f7521370fe40ac89b3f93e9e5f5072f0192de7
# digest: 4a0a004730450221008132561626bc3ef36822cb33518b731d96056a898165966920163c60088aca8a022030f7ca08e18d24f031d511fdb89dd8fd1e83a681bdc67dd062bd47039132f911

View File

@ -18,4 +18,4 @@ code:
- type: word
words:
- "hello from input"
# digest: 490a00463044022037b485132a0ff1f52bde1ce71908fd692d93d9cda15d5b70275c7fc89a0aa257022043f36ca1c7062d4d74dde377e5a0065c1507bfcad2a8389772eebfa99fd16af2
# digest: 4b0a00483046022100dd46d2316163dd5f073bdf84d038958114c6a00914e737f0daa12827994eaa7a022100c8a6bbdedd0c6dc315e6c61f98dc3add9121b8ab340d333401dc58962284fc9a

View File

@ -26,4 +26,4 @@ code:
part: interactsh_protocol
words:
- "http"
# digest: 4a0a004730450221008d0e317bd37ab7dfdbaf3d26ed2072521f0ea00a61d45d0e6d592886faeac1840220410070cd29ffa05113b387c5afcd13fd65dab2b67f21583a1aeddb352ab5aa88
# digest: 4b0a00483046022100b5084304ca60c6c7d89e0a5f23ed82a26f59cb2c8ccb3a90535792d4d77cd80d022100eea2c5a3164f83a9b0bcf60e637e7a710358cef7a96c0fc016185cce3f23d6a4

View File

@ -20,4 +20,4 @@ code:
- type: word
words:
- "hello from input"
# digest: 4b0a00483046022100b7fef4d710fa7e3b8916dc990a381551deb42c558d92a043358325f3994078f5022100db47c9f752185ad42af1a5268788567f88fcd8e41053674fdb637f01f02bbbaa
# digest: 4a0a00473045022030a0b1fddd6c5ac0c5d217eef447c7ea54e69a044eb12376d06d5c5aa8171f67022100bb150ff1bf3b3ee0dead7ffec6cc038c860f0f660df1fc5e61eed871a439d6f4

View File

@ -19,7 +19,7 @@ func (h *customConfigDirTest) Execute(filePath string) error {
return err
}
defer os.RemoveAll(customTempDirectory)
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "8x8exch02.8x8.com", debug, "-config-directory", customTempDirectory)
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, []string{"NUCLEI_CONFIG_DIR=" + customTempDirectory}, "-t", filePath, "-u", "8x8exch02.8x8.com")
if err != nil {
return err
}

View File

@ -55,7 +55,7 @@ func (h *remoteTemplateList) Execute(templateList string) error {
}
defer os.Remove("test-config.yaml")
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list", "-config", "test-config.yaml")
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/template_list", "-config", "test-config.yaml")
if err != nil {
return err
}
@ -78,7 +78,7 @@ func (h *excludedTemplate) Execute(templateList string) error {
ts := httptest.NewServer(router)
defer ts.Close()
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-t", templateList, "-include-templates", templateList)
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-t", templateList, "-include-templates", templateList)
if err != nil {
return err
}
@ -112,7 +112,7 @@ func (h *remoteTemplateListNotAllowed) Execute(templateList string) error {
ts := httptest.NewServer(router)
defer ts.Close()
_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list")
_, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/template_list")
if err == nil {
return fmt.Errorf("expected error for not allowed remote template list url")
}
@ -154,7 +154,7 @@ func (h *remoteWorkflowList) Execute(workflowList string) error {
}
defer os.Remove("test-config.yaml")
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/workflow_list", "-config", "test-config.yaml")
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/workflow_list", "-config", "test-config.yaml")
if err != nil {
return err
}
@ -170,7 +170,7 @@ func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error
ts := httptest.NewServer(router)
defer ts.Close()
_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/404")
_, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/404")
if err == nil {
return fmt.Errorf("expected error for nonexisting workflow url")
}
@ -186,7 +186,7 @@ func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error
ts := httptest.NewServer(router)
defer ts.Close()
_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/404")
_, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/404")
if err == nil {
return fmt.Errorf("expected error for nonexisting workflow url")
}

View File

@ -262,7 +262,6 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringVarP(&options.Interface, "interface", "i", "", "network interface to use for network scan"),
flagSet.StringVarP(&options.AttackType, "attack-type", "at", "", "type of payload combinations to perform (batteringram,pitchfork,clusterbomb)"),
flagSet.StringVarP(&options.SourceIP, "source-ip", "sip", "", "source ip address to use for network scan"),
flagSet.StringVar(&options.CustomConfigDir, "config-directory", "", "override the default config path ($home/.config)"),
flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 10*1024*1024, "max response size to read in bytes"),
flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", 1*1024*1024, "max response size to read in bytes"),
flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
@ -388,7 +387,11 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.BoolVar(&options.NoTables, "no-tables", false, "do not display pretty-printed tables"),
flagSet.IntVar(&options.OutputLimit, "limit", 100, "limit the number of output to display"),
)
// nuclei has multiple migrations
// ex: resume.cfg moved to platform standard cache dir from config dir
// ex: config.yaml moved to platform standard config dir from linux specific config dir
// and hence it will be attempted in config package during init
goflags.DisableAutoConfigMigration = true
_ = flagSet.Parse()
gologger.DefaultLogger.SetTimestamp(options.Timestamp, levels.LevelDebug)
@ -401,8 +404,8 @@ on extensive configurability, massive extensibility and ease of use.`)
if options.LeaveDefaultPorts {
http.LeaveDefaultPorts = true
}
if options.CustomConfigDir != "" {
config.DefaultConfig.SetConfigDir(options.CustomConfigDir)
if customConfigDir := os.Getenv(config.NucleiConfigDirEnv); customConfigDir != "" {
config.DefaultConfig.SetConfigDir(customConfigDir)
readFlagsConfig(flagSet)
}
if cfgFile != "" {
@ -424,7 +427,7 @@ on extensive configurability, massive extensibility and ease of use.`)
// cleanupOldResumeFiles cleans up resume files older than 10 days.
func cleanupOldResumeFiles() {
root := config.DefaultConfig.GetConfigDir()
root := config.DefaultConfig.GetCacheDir()
filter := fileutil.FileFilters{
OlderThan: 24 * time.Hour * 10, // cleanup on the 10th day
Prefix: "resume-",
@ -469,6 +472,8 @@ func disableUpdatesCallback() {
// printVersion prints the nuclei version and exits.
func printVersion() {
gologger.Info().Msgf("Nuclei Engine Version: %s", config.Version)
gologger.Info().Msgf("Nuclei Config Directory: %s", config.DefaultConfig.GetConfigDir())
gologger.Info().Msgf("Nuclei Cache Directory: %s", config.DefaultConfig.GetCacheDir()) // cache dir contains resume files
os.Exit(0)
}

View File

@ -39,7 +39,7 @@ require (
github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222
github.com/xanzy/go-gitlab v0.84.0
go.uber.org/multierr v1.11.0
golang.org/x/net v0.15.0
golang.org/x/net v0.16.0
golang.org/x/oauth2 v0.11.0
golang.org/x/text v0.13.0
gopkg.in/yaml.v2 v2.4.0
@ -79,7 +79,7 @@ require (
github.com/praetorian-inc/fingerprintx v1.1.9
github.com/projectdiscovery/dsl v0.0.22-0.20230911020052-7ab80c9abba8
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/goflags v0.1.20
github.com/projectdiscovery/goflags v0.1.24-0.20231009194911-044c556377a1
github.com/projectdiscovery/gologger v1.1.11
github.com/projectdiscovery/gostruct v0.0.1
github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6
@ -91,7 +91,7 @@ require (
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.4
github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1
github.com/projectdiscovery/utils v0.0.54
github.com/projectdiscovery/utils v0.0.58-0.20231009161115-60268dca6e8f
github.com/projectdiscovery/wappalyzergo v0.0.107
github.com/redis/go-redis/v9 v9.1.0
github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02
@ -128,6 +128,7 @@ require (
github.com/cloudflare/cfssl v1.6.4 // indirect
github.com/cloudflare/circl v1.3.3 // indirect
github.com/containerd/continuity v0.4.2 // indirect
github.com/denisbrodbeck/machineid v1.0.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dlclark/regexp2 v1.10.0 // indirect
github.com/docker/cli v24.0.5+incompatible // indirect
@ -276,12 +277,12 @@ require (
go.etcd.io/bbolt v1.3.7 // indirect
go.uber.org/zap v1.25.0 // indirect
goftp.io/server/v2 v2.0.1 // indirect
golang.org/x/crypto v0.13.0
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
golang.org/x/mod v0.12.0 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/crypto v0.14.0
golang.org/x/exp v0.0.0-20231006140011-7918f672742d
golang.org/x/mod v0.13.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.13.0 // indirect
golang.org/x/tools v0.14.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect

View File

@ -243,6 +243,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw=
github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
@ -778,8 +780,8 @@ github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOi
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
github.com/projectdiscovery/goflags v0.1.20 h1:38JgrMOJWg+7G3GUvZJ5cpblBsOFgftNihFusv57A+4=
github.com/projectdiscovery/goflags v0.1.20/go.mod h1:HlJeOmWiMRmjg7PnOMiQkwgLMs88DKgymV1motcxFZw=
github.com/projectdiscovery/goflags v0.1.24-0.20231009194911-044c556377a1 h1:nigd9/uIhKwgFg1PYoXwRbn8cO0XOXiHhPXh3X0bIRc=
github.com/projectdiscovery/goflags v0.1.24-0.20231009194911-044c556377a1/go.mod h1:15Hc92q689vj6bJv0NdLb9NT/PTrPtT9ItSZBUAJULs=
github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8=
github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY=
github.com/projectdiscovery/gostruct v0.0.1 h1:1KvR6Pn4mDbQqoLEQzhRfHpbreLno2R9xqRCCt5tgmU=
@ -815,8 +817,10 @@ github.com/projectdiscovery/tlsx v1.1.4 h1:jXRvichO/ZfhYERch1CbNS1PRbS2KgSBj7JoW
github.com/projectdiscovery/tlsx v1.1.4/go.mod h1:crzMlxOokVQDwGVm51JPZi1ZAgzxhNl1KVRmbff6pkI=
github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak=
github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8=
github.com/projectdiscovery/utils v0.0.54 h1:qwTIalrK8pKYaxFObdeSfCtwDmVCN9qswc8+7jIpnBM=
github.com/projectdiscovery/utils v0.0.54/go.mod h1:WhzbWSyGkTDn4Jvw+7jM2yP675/RARegNjoA6S7zYcc=
github.com/projectdiscovery/utils v0.0.58-0.20231009151631-3681bca54127 h1:GNsAR3SPNcWE3wpW+HJJsPOGWnmdMdECXZN3k3zqkTI=
github.com/projectdiscovery/utils v0.0.58-0.20231009151631-3681bca54127/go.mod h1:5ub86JF91NnI3nTMIzEpL/pfsNb0jtHznzKi9hv03X4=
github.com/projectdiscovery/utils v0.0.58-0.20231009161115-60268dca6e8f h1:5GMMQ6d7vqLMvjfibclgWgptj7vm9iDAz8xgRCYd+iI=
github.com/projectdiscovery/utils v0.0.58-0.20231009161115-60268dca6e8f/go.mod h1:5ub86JF91NnI3nTMIzEpL/pfsNb0jtHznzKi9hv03X4=
github.com/projectdiscovery/wappalyzergo v0.0.107 h1:B8gzJpAh08f1o+OiDunHAfKtqXiDnFCc7Rj1qKp+DB8=
github.com/projectdiscovery/wappalyzergo v0.0.107/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@ -1107,8 +1111,8 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -1119,8 +1123,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -1143,8 +1147,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -1200,8 +1204,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos=
golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -1226,7 +1230,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -1300,8 +1304,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@ -1311,7 +1315,7 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -1386,8 +1390,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

View File

@ -16,6 +16,29 @@ import (
"github.com/projectdiscovery/ratelimit"
)
// TemplateSources contains template sources
// which define where to load templates from
type TemplateSources struct {
Templates []string // template file/directory paths
Workflows []string // workflow file/directory paths
RemoteTemplates []string // remote template urls
RemoteWorkflows []string // remote workflow urls
TrustedDomains []string // trusted domains for remote templates/workflows
}
// WithTemplatesOrWorkflows sets templates / workflows to use /load
func WithTemplatesOrWorkflows(sources TemplateSources) NucleiSDKOptions {
return func(e *NucleiEngine) error {
// by default all of these values are empty
e.opts.Templates = sources.Templates
e.opts.Workflows = sources.Workflows
e.opts.TemplateURLs = sources.RemoteTemplates
e.opts.WorkflowURLs = sources.RemoteWorkflows
e.opts.RemoteTemplateDomainList = append(e.opts.RemoteTemplateDomainList, sources.TrustedDomains...)
return nil
}
}
// config contains all SDK configuration options
type TemplateFilters struct {
Severity string // filter by severities (accepts CSV values of info, low, medium, high, critical)

View File

@ -41,6 +41,7 @@ func ExampleThreadSafeNucleiEngine() {
}
// setup sizedWaitgroup to handle concurrency
// here we are using sizedWaitgroup to limit concurrency to 1
// but can be anything in general
sg := sizedwaitgroup.New(1)
// scan 1 = run dns templates on scanme.sh
@ -71,7 +72,6 @@ func ExampleThreadSafeNucleiEngine() {
// Output:
// [nameserver-fingerprint] scanme.sh
// [dns-saas-service-detection] honey.scanme.sh
}
func TestMain(m *testing.M) {

View File

@ -20,6 +20,24 @@ func TestSimpleNuclei(t *testing.T) {
defer ne.Close()
}
func TestSimpleNucleiRemote(t *testing.T) {
ne, err := nuclei.NewNucleiEngine(
nuclei.WithTemplatesOrWorkflows(
nuclei.TemplateSources{
RemoteTemplates: []string{"https://templates.nuclei.sh/public/nameserver-fingerprint.yaml"},
},
),
)
require.Nil(t, err)
ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here
err = ne.LoadAllTemplates()
require.Nil(t, err, "could not load templates")
// when callback is nil it nuclei will print JSON output to stdout
err = ne.ExecuteWithCallback(nil)
require.Nil(t, err)
defer ne.Close()
}
func TestThreadSafeNuclei(t *testing.T) {
// create nuclei engine with options
ne, err := nuclei.NewThreadSafeNucleiEngine()

View File

@ -23,6 +23,9 @@ const (
CustomGitHubTemplatesDirName = "github"
CustomAzureTemplatesDirName = "azure"
CustomGitLabTemplatesDirName = "gitlab"
BinaryName = "nuclei"
FallbackConfigFolderName = ".nuclei-config"
NucleiConfigDirEnv = "NUCLEI_CONFIG_DIR"
)
// IsOutdatedVersion compares two versions and returns true

View File

@ -5,10 +5,12 @@ import (
"crypto/md5"
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/gologger"
errorutil "github.com/projectdiscovery/utils/errors"
fileutil "github.com/projectdiscovery/utils/file"
@ -158,6 +160,14 @@ func (c *Config) GetNewAdditions() []string {
return arr
}
// GetCacheDir returns the nuclei cache directory
// with new version of nuclei cache directory is changed
// instead of saving resume files in nuclei config directory
// they are saved in nuclei cache directory
func (c *Config) GetCacheDir() string {
return folderutil.AppCacheDirOrDefault(".nuclei-cache", BinaryName)
}
// SetConfigDir sets the nuclei configuration directory
// and appropriate changes are made to the config
func (c *Config) SetConfigDir(dir string) {
@ -276,15 +286,24 @@ func (c *Config) copyIgnoreFile() {
}
ignoreFilePath := c.GetIgnoreFilePath()
if !fileutil.FileExists(ignoreFilePath) {
// copy ignore file
if err := fileutil.CopyFile(filepath.Join(getDefaultConfigDir(), NucleiIgnoreFileName), ignoreFilePath); err != nil {
// copy ignore file from default config directory
if err := fileutil.CopyFile(filepath.Join(folderutil.AppConfigDirOrDefault(FallbackConfigFolderName, BinaryName), NucleiIgnoreFileName), ignoreFilePath); err != nil {
gologger.Error().Msgf("Could not copy nuclei ignore file at %s: %s", ignoreFilePath, err)
}
}
}
func init() {
ConfigDir := getDefaultConfigDir()
// first attempt to migrate all files from old config directory to new config directory
goflags.AttemptConfigMigration() // regardless how many times this is called it will only migrate once based on condition
ConfigDir := folderutil.AppConfigDirOrDefault(FallbackConfigFolderName, BinaryName)
if cfgDir := os.Getenv(NucleiConfigDirEnv); cfgDir != "" {
ConfigDir = cfgDir
}
// create config directory if not exists
if !fileutil.FolderExists(ConfigDir) {
if err := fileutil.CreateFolder(ConfigDir); err != nil {
gologger.Error().Msgf("failed to create config directory at %v got: %s", ConfigDir, err)
@ -303,6 +322,9 @@ func init() {
gologger.Error().Msgf("failed to write config file at %s got: %s", DefaultConfig.getTemplatesConfigFilePath(), err)
}
}
// attempt to migrate resume files
// this also happens once regardless of how many times this is called
migrateResumeFiles()
// Loads/updates paths of custom templates
// Note: custom templates paths should not be updated in config file
// and even if it is changed we don't follow it since it is not expected behavior
@ -310,22 +332,67 @@ func init() {
DefaultConfig.SetTemplatesDir(DefaultConfig.TemplatesDirectory)
}
func getDefaultConfigDir() string {
// Review Needed: Earlier a dependency was used to locate home dir
// i.e "github.com/mitchellh/go-homedir" not sure if it is needed
// Even if such case exists it should be abstracted via below function call in utils/folder
homedir := folderutil.HomeDirOrDefault("")
// TBD: we should probably stick to specification and use config directories provided by distro
// instead of manually creating one since $HOME/.config/ is config directory of Linux desktops
// Ref: https://pkg.go.dev/os#UserConfigDir
// some distros like NixOS or others have totally different config directories this causes issues for us (since we are not using os.UserConfigDir)
userCfgDir := filepath.Join(homedir, ".config")
return filepath.Join(userCfgDir, "nuclei")
}
// Add Default Config adds default when .templates-config.json file is not present
func applyDefaultConfig() {
DefaultConfig.TemplatesDirectory = filepath.Join(DefaultConfig.homeDir, NucleiTemplatesDirName)
// updates all necessary paths
DefaultConfig.SetTemplatesDir(DefaultConfig.TemplatesDirectory)
}
func migrateResumeFiles() {
// attempt to migrate old resume files to new directory structure
// after migration has been done in goflags
oldResumeDir := DefaultConfig.GetConfigDir()
// migrate old resume file to new directory structure
if !fileutil.FileOrFolderExists(DefaultConfig.GetCacheDir()) && fileutil.FileOrFolderExists(oldResumeDir) {
// this means new cache dir doesn't exist, so we need to migrate
// first check if old resume file exists if not then no need to migrate
exists := false
files, err := os.ReadDir(oldResumeDir)
if err != nil {
// log silently
log.Printf("could not read old resume dir: %s\n", err)
return
}
for _, file := range files {
if strings.HasSuffix(file.Name(), ".cfg") {
exists = true
break
}
}
if !exists {
// no need to migrate
return
}
// create new cache dir
err = os.MkdirAll(DefaultConfig.GetCacheDir(), os.ModePerm)
if err != nil {
// log silently
log.Printf("could not create new cache dir: %s\n", err)
return
}
err = filepath.WalkDir(oldResumeDir, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
if !strings.HasSuffix(path, ".cfg") {
return nil
}
err = os.Rename(path, filepath.Join(DefaultConfig.GetCacheDir(), filepath.Base(path)))
if err != nil {
return err
}
return nil
})
if err != nil {
// log silently
log.Printf("could not migrate old resume files: %s\n", err)
return
}
}
}

View File

@ -32,6 +32,10 @@ const (
httpsPrefix = "https://"
)
var (
TrustedTemplateDomains = []string{"templates.nuclei.sh", "cloud.projectdiscovery.io"}
)
// Config contains the configuration options for the loader
type Config struct {
Templates []string
@ -100,6 +104,7 @@ func NewConfig(options *types.Options, catalog catalog.Catalog, executerOpts pro
Catalog: catalog,
ExecutorOptions: executerOpts,
}
loaderConfig.RemoteTemplateDomainList = append(loaderConfig.RemoteTemplateDomainList, TrustedTemplateDomains...)
return &loaderConfig
}

View File

@ -37,7 +37,7 @@ func RunNucleiAndGetResults(isTemplate bool, template, url string, debug bool, e
templateOrWorkflowFlag = "-w"
}
return RunNucleiBareArgsAndGetResults(debug, append([]string{
return RunNucleiBareArgsAndGetResults(debug, nil, append([]string{
templateOrWorkflowFlag,
template,
"-target",
@ -45,7 +45,7 @@ func RunNucleiAndGetResults(isTemplate bool, template, url string, debug bool, e
}, extra...)...)
}
func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, error) {
func RunNucleiBareArgsAndGetResults(debug bool, env []string, extra ...string) ([]string, error) {
cmd := exec.Command("./nuclei")
extra = append(extra, ExtraDebugArgs...)
cmd.Args = append(cmd.Args, extra...)
@ -53,6 +53,9 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro
cmd.Args = append(cmd.Args, "-interactions-poll-duration", "1")
cmd.Args = append(cmd.Args, "-interactions-cooldown-period", "10")
cmd.Args = append(cmd.Args, "-allow-local-file-access")
if env != nil {
cmd.Env = append(os.Environ(), env...)
}
if debug {
cmd.Args = append(cmd.Args, "-debug")
cmd.Stderr = os.Stderr

View File

@ -14,7 +14,7 @@ import (
const DefaultResumeFileName = "resume-%s.cfg"
func DefaultResumeFilePath() string {
configDir := config.DefaultConfig.GetConfigDir()
configDir := config.DefaultConfig.GetCacheDir()
resumeFile := filepath.Join(configDir, fmt.Sprintf(DefaultResumeFileName, xid.New().String()))
return resumeFile
}

View File

@ -329,8 +329,6 @@ type Options struct {
DisableStdin bool
// IncludeConditions is the list of conditions templates should match
IncludeConditions goflags.StringSlice
// Custom Config Directory
CustomConfigDir string
// Enable uncover engine
Uncover bool
// Uncover search query